tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[feat] reindex added to admin

Browse Source
This commit is contained in:
Thad Miller
2026-05-28 15:14:18 -04:00
parent ae0f3d6683
commit b0dbe7ced5
7 changed files with 648 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

832
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
package.json
View File

@@ -20,7 +20,7 @@
"chart.js": "^4.5.1",
"csv": "^6.4.1",
"dotenv": "^17.2.4",
"drizzle-orm": "^1.0.0-beta.15-859cf75",
"drizzle-orm": "1.0.0-beta.15-859cf75",
"pg": "^8.20.0",
"sass": "^1.97.3",
"typesense": "^3.0.1"
@@ -29,7 +29,7 @@
"@types/bootstrap": "^5.2.10",
"@types/node": "^25.2.1",
"@types/pg": "^8.18.0",
"drizzle-kit": "^1.0.0-beta.15-859cf75",
"drizzle-kit": "1.0.0-beta.15-859cf75",
"typescript": "^5.9.3"
}
}

27
scripts/pokemon-helper.ts
View File

@@ -12,6 +12,9 @@ const DollarToInt = (dollar: any) => {
return Math.round(dollar * 100);
}
export type Logger = (msg: string) => void;
const defaultLogger: Logger = (msg) => console.log(chalk.green(msg));
export const Sleep = (ms: number) => {
@@ -39,7 +42,7 @@ export const GetNumberOrNull = (value: any): number | null => {
// Delete and recreate the 'cards' index
export const createCardCollection = async () => {
export const createCardCollection = async (log: Logger = defaultLogger) => {
try {
await client.collections('cards').delete();
} catch (error) {
@@ -68,11 +71,11 @@ export const createCardCollection = async () => {
// { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true }
],
});
console.log(chalk.green('Collection "cards" created successfully.'));
log('Collection "cards" created successfully.');
}
// Delete and recreate the 'skus' index
export const createSkuCollection = async () => {
export const createSkuCollection = async (log: Logger = defaultLogger) => {
try {
await client.collections('skus').delete();
} catch (error) {
@@ -89,11 +92,11 @@ export const createSkuCollection = async () => {
{ name: 'card_id', type: 'string', reference: 'cards.id', async_reference: true },
]
});
console.log(chalk.green('Collection "skus" created successfully.'));
log('Collection "skus" created successfully.');
}
// Delete and recreate the 'inventory' index
export const createInventoryCollection = async () => {
export const createInventoryCollection = async (log: Logger = defaultLogger) => {
try {
await client.collections('inventories').delete();
} catch (error) {
@@ -117,11 +120,11 @@ export const createInventoryCollection = async () => {
{ name: 'cardType', type: 'string' },
]
});
console.log(chalk.green('Collection "inventories" created successfully.'));
log('Collection "inventories" created successfully.');
}
export const upsertCardCollection = async (db:DBInstance) => {
export const upsertCardCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
const pokemon = await db.query.cards.findMany({
with: { set: true, tcgdata: true, prices: true },
});
@@ -157,10 +160,10 @@ export const upsertCardCollection = async (db:DBInstance) => {
// sku_id: card.prices.map(price => price.skuId.toString())
};
}), { action: 'upsert' });
console.log(chalk.green('Collection "cards" indexed successfully.'));
log('Collection "cards" indexed successfully.');
}
export const upsertSkuCollection = async (db:DBInstance) => {
export const upsertSkuCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
const skus = await db.query.skus.findMany();
await client.collections('skus').documents().import(skus.map(sku => ({
id: sku.skuId.toString(),
@@ -170,10 +173,10 @@ export const upsertSkuCollection = async (db:DBInstance) => {
marketPrice: DollarToInt(sku.marketPrice),
card_id: sku.cardId.toString(),
})), { action: 'upsert' });
console.log(chalk.green('Collection "skus" indexed successfully.'));
log('Collection "skus" indexed successfully.');
}
export const upsertInventoryCollection = async (db:DBInstance) => {
export const upsertInventoryCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
const inv = await db.query.inventory.findMany({
with: { sku: { with: { card: { with: { set: true } } } } }
});
@@ -198,7 +201,7 @@ export const upsertInventoryCollection = async (db:DBInstance) => {
i.sku?.card?.artist || ""
].join(' '),
})), { action: 'upsert' });
console.log(chalk.green('Collection "inventories" indexed successfully.'));
log('Collection "inventories" indexed successfully.');
}

2
src/assets/css/_bootstrap.scss
View File

@@ -18,7 +18,7 @@
@import 'bootstrap/scss/containers';
@import 'bootstrap/scss/images';
@import 'bootstrap/scss/nav';
// @import 'bootstrap/scss/accordion';
@import 'bootstrap/scss/accordion';
@import 'bootstrap/scss/alert';
@import 'bootstrap/scss/badge';
// @import 'bootstrap/scss/breadcrumb';

4
src/middleware.ts
View File

@@ -10,8 +10,8 @@ declare global {
}
const isProtectedRoute = createRouteMatcher(['/pokemon']);
const isAdminRoute = createRouteMatcher(['/admin']);
const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex']);
const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK";
const ADMIN_ORG_IDS = new Set([
"org_3Baav9czkRLLlC7g89oJWqRRulK",

180
src/pages/admin.astro
View File

@@ -9,10 +9,182 @@ import Footer from '../components/Footer.astro';
<NavBar slot="navbar">
<NavItems slot="navItems" />
</NavBar>
<div class="row mb-4" slot="page">
<div class="col-12">
<h1>Admin Panel</h1>
<div slot="page">
<div class="container my-4">
<div class="row mb-4">
<div class="col-12">
<h1>Admin Panel</h1>
<div class="accordion" id="adminAccordion">
<div class="accordion-item">
<h2 class="accordion-header" id="reindexHeading">
<button class="accordion-button collapsed" type="button" data-bs-toggle="collapse"
data-bs-target="#reindexCollapse" aria-expanded="false" aria-controls="reindexCollapse">
Reindex
</button>
</h2>
<div id="reindexCollapse" class="accordion-collapse collapse" aria-labelledby="reindexHeading"
data-bs-parent="#adminAccordion">
<div class="accordion-body">
<form id="reindexForm">
<div class="mb-3">
<div class="form-text mb-2">Select collections to reindex:</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="reindexCards" name="cards" checked />
<label class="form-check-label" for="reindexCards">Cards</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="reindexSkus" name="skus" checked />
<label class="form-check-label" for="reindexSkus">SKUs</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="reindexInventory" name="inventory" checked />
<label class="form-check-label" for="reindexInventory">Inventory</label>
</div>
</div>
<div class="mb-3 form-check">
<input class="form-check-input" type="checkbox" id="reindexRecreate" name="recreate" />
<label class="form-check-label" for="reindexRecreate">
Recreate index (drops and recreates collections; otherwise updates in place)
</label>
</div>
<button type="submit" class="btn btn-purple" id="reindexRun">Run Reindex</button>
</form>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Reusable scrollable progress modal. Open via window.AdminProgress.open(title). -->
<div class="modal fade" id="adminProgressModal" tabindex="-1" aria-labelledby="adminProgressLabel" aria-hidden="true"
data-bs-backdrop="static" data-bs-keyboard="false">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="adminProgressLabel">Progress</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"
id="adminProgressClose" disabled></button>
</div>
<div class="modal-body p-0">
<pre id="adminProgressLog"
class="m-0 p-3 small"
style="max-height: 60vh; overflow-y: auto; white-space: pre-wrap; word-break: break-word;"></pre>
</div>
<div class="modal-footer">
<span class="me-auto small text-secondary" id="adminProgressStatus">Idle</span>
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal"
id="adminProgressDismiss" disabled>Close</button>
</div>
</div>
</div>
</div>
</div>
<Footer slot="footer" />
</Layout>
</Layout>
<script>
// Reusable progress modal. Other admin features can call window.AdminProgress.
type ProgressHandle = {
append: (line: string) => void;
setStatus: (text: string) => void;
done: (text?: string) => void;
};
declare global {
interface Window {
AdminProgress: {
open: (title: string) => Promise<ProgressHandle>;
};
}
}
const getBootstrap = (): any => (window as any).bootstrap;
const openProgress = async (title: string): Promise<ProgressHandle> => {
const modalEl = document.getElementById('adminProgressModal')!;
const labelEl = document.getElementById('adminProgressLabel')!;
const logEl = document.getElementById('adminProgressLog')!;
const statusEl = document.getElementById('adminProgressStatus')!;
const closeBtn = document.getElementById('adminProgressClose') as HTMLButtonElement;
const dismissBtn = document.getElementById('adminProgressDismiss') as HTMLButtonElement;
labelEl.textContent = title;
logEl.textContent = '';
statusEl.textContent = 'Running...';
closeBtn.disabled = true;
dismissBtn.disabled = true;
const modal = getBootstrap().Modal.getOrCreateInstance(modalEl);
modal.show();
return {
append: (line: string) => {
logEl.textContent += (logEl.textContent ? '\n' : '') + line;
logEl.scrollTop = logEl.scrollHeight;
},
setStatus: (text: string) => { statusEl.textContent = text; },
done: (text = 'Done') => {
statusEl.textContent = text;
closeBtn.disabled = false;
dismissBtn.disabled = false;
},
};
};
window.AdminProgress = { open: openProgress };
// Reindex form wiring
const form = document.getElementById('reindexForm') as HTMLFormElement | null;
if (form) {
form.addEventListener('submit', async (e) => {
e.preventDefault();
const runBtn = document.getElementById('reindexRun') as HTMLButtonElement;
const body = {
cards: (document.getElementById('reindexCards') as HTMLInputElement).checked,
skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked,
inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked,
recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked,
};
runBtn.disabled = true;
const progress = await window.AdminProgress.open('Reindex');
try {
const resp = await fetch('/api/reindex', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (!resp.ok || !resp.body) {
progress.append(`Request failed: ${resp.status} ${resp.statusText}`);
progress.done('Failed');
return;
}
const reader = resp.body.getReader();
const decoder = new TextDecoder();
let buf = '';
while (true) {
const { value, done } = await reader.read();
if (done) break;
buf += decoder.decode(value, { stream: true });
const lines = buf.split('\n');
buf = lines.pop() ?? '';
for (const line of lines) progress.append(line);
}
if (buf) progress.append(buf);
progress.done('Done');
} catch (err: any) {
progress.append(`Error: ${err?.message || String(err)}`);
progress.done('Failed');
} finally {
runBtn.disabled = false;
}
});
}
</script>

71
src/pages/api/reindex.ts Normal file
View File

@@ -0,0 +1,71 @@
import type { APIRoute } from 'astro';
import { db } from '../../db/index';
import * as Indexing from '../../../scripts/pokemon-helper';
export const POST: APIRoute = async ({ request }) => {
const { cards, skus, inventory, recreate } = await request.json().catch(() => ({} as any));
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
const log = (msg: string) => {
controller.enqueue(encoder.encode(msg + '\n'));
};
try {
if (!cards && !skus && !inventory) {
log('No collections selected.');
return;
}
if (recreate) {
if (cards) {
log('Recreating "cards" collection...');
await Indexing.createCardCollection(log);
}
if (skus) {
log('Recreating "skus" collection...');
await Indexing.createSkuCollection(log);
}
if (inventory) {
log('Recreating "inventories" collection...');
await Indexing.createInventoryCollection(log);
}
}
if (cards) {
log('Indexing "cards"...');
await Indexing.upsertCardCollection(db, log);
}
if (skus) {
log('Indexing "skus"...');
await Indexing.upsertSkuCollection(db, log);
}
if (inventory) {
log('Indexing "inventories"...');
await Indexing.upsertInventoryCollection(db, log);
}
log('Reindex complete.');
} catch (e: any) {
const cause = e?.cause;
const causeMsg = cause?.message || (cause ? String(cause) : '');
const causeDetail = cause?.detail ? ` | detail: ${cause.detail}` : '';
const causeCode = cause?.code ? ` | code: ${cause.code}` : '';
log(`Error: ${e?.message || String(e)}`);
if (causeMsg) log(`Caused by: ${causeMsg}${causeCode}${causeDetail}`);
console.error('Reindex error:', e);
} finally {
controller.close();
}
},
});
return new Response(stream, {
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'no-cache, no-transform',
'X-Accel-Buffering': 'no',
},
});
};