[feat] reindex added to admin

package.json

@@ -20,7 +20,7 @@
     "chart.js": "^4.5.1",
     "csv": "^6.4.1",
     "dotenv": "^17.2.4",
-    "drizzle-orm": "^1.0.0-beta.15-859cf75",
+    "drizzle-orm": "1.0.0-beta.15-859cf75",
     "pg": "^8.20.0",
     "sass": "^1.97.3",
     "typesense": "^3.0.1"
@@ -29,7 +29,7 @@
     "@types/bootstrap": "^5.2.10",
     "@types/node": "^25.2.1",
     "@types/pg": "^8.18.0",
-    "drizzle-kit": "^1.0.0-beta.15-859cf75",
+    "drizzle-kit": "1.0.0-beta.15-859cf75",
     "typescript": "^5.9.3"
   }
 }

scripts/pokemon-helper.ts

@@ -12,6 +12,9 @@ const DollarToInt = (dollar: any) => {
     return Math.round(dollar * 100);
 }
 
+export type Logger = (msg: string) => void;
+const defaultLogger: Logger = (msg) => console.log(chalk.green(msg));
+
 
 
 export const Sleep = (ms: number) => {
@@ -39,7 +42,7 @@ export const GetNumberOrNull = (value: any): number | null => {
 
 
 // Delete and recreate the 'cards' index
-export const createCardCollection = async () => {
+export const createCardCollection = async (log: Logger = defaultLogger) => {
     try {
         await client.collections('cards').delete();
     } catch (error) {
@@ -68,11 +71,11 @@ export const createCardCollection = async () => {
             // { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true }
         ],
     });
-    console.log(chalk.green('Collection "cards" created successfully.'));
+    log('Collection "cards" created successfully.');
 }
 
 // Delete and recreate the 'skus' index
-export const createSkuCollection = async () => {
+export const createSkuCollection = async (log: Logger = defaultLogger) => {
     try {
         await client.collections('skus').delete();
     } catch (error) {
@@ -89,11 +92,11 @@ export const createSkuCollection = async () => {
             { name: 'card_id', type: 'string', reference: 'cards.id', async_reference: true },
         ]
     });
-    console.log(chalk.green('Collection "skus" created successfully.'));
+    log('Collection "skus" created successfully.');
 }
 
 // Delete and recreate the 'inventory' index
-export const createInventoryCollection = async () => {
+export const createInventoryCollection = async (log: Logger = defaultLogger) => {
     try {
         await client.collections('inventories').delete();
     } catch (error) {
@@ -117,11 +120,11 @@ export const createInventoryCollection = async () => {
             { name: 'cardType', type: 'string' },
         ]
     });
-    console.log(chalk.green('Collection "inventories" created successfully.'));
+    log('Collection "inventories" created successfully.');
 }
 
 
-export const upsertCardCollection = async (db:DBInstance) => {
+export const upsertCardCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
     const pokemon = await db.query.cards.findMany({
         with: { set: true, tcgdata: true, prices: true },
     });
@@ -157,10 +160,10 @@ export const upsertCardCollection = async (db:DBInstance) => {
             // sku_id: card.prices.map(price => price.skuId.toString())
         };
     }), { action: 'upsert' });
-    console.log(chalk.green('Collection "cards" indexed successfully.'));
+    log('Collection "cards" indexed successfully.');
 }
 
-export const upsertSkuCollection = async (db:DBInstance) => {
+export const upsertSkuCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
     const skus = await db.query.skus.findMany();
     await client.collections('skus').documents().import(skus.map(sku => ({
         id: sku.skuId.toString(),
@@ -170,10 +173,10 @@ export const upsertSkuCollection = async (db:DBInstance) => {
         marketPrice: DollarToInt(sku.marketPrice),
         card_id: sku.cardId.toString(),
     })), { action: 'upsert' }); 
-    console.log(chalk.green('Collection "skus" indexed successfully.'));
+    log('Collection "skus" indexed successfully.');
 }
 
-export const upsertInventoryCollection = async (db:DBInstance) => {
+export const upsertInventoryCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
     const inv = await db.query.inventory.findMany({
         with: { sku: { with: { card: { with: { set: true } } } } }
     });
@@ -198,7 +201,7 @@ export const upsertInventoryCollection = async (db:DBInstance) => {
             i.sku?.card?.artist || ""
         ].join(' '),
     })), { action: 'upsert' });
-    console.log(chalk.green('Collection "inventories" indexed successfully.'));
+    log('Collection "inventories" indexed successfully.');
 }
 
 

src/assets/css/_bootstrap.scss

@@ -18,7 +18,7 @@
 @import 'bootstrap/scss/containers';
 @import 'bootstrap/scss/images';
 @import 'bootstrap/scss/nav';
-// @import 'bootstrap/scss/accordion';
+@import 'bootstrap/scss/accordion';
 @import 'bootstrap/scss/alert';
 @import 'bootstrap/scss/badge';
 // @import 'bootstrap/scss/breadcrumb';

src/middleware.ts

@@ -1,5 +1,4 @@
 import { clerkMiddleware, createRouteMatcher, clerkClient } from '@clerk/astro/server';
-import type { MiddlewareNext } from 'astro';
 import 'dotenv/config';
  
 declare global {
@@ -11,9 +10,13 @@ declare global {
 }
  
 const isProtectedRoute = createRouteMatcher(['/pokemon']);
-const isAdminRoute = createRouteMatcher(['/admin']);
+const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex']);
 
 const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK";
+const ADMIN_ORG_IDS = new Set([
+  "org_3Baav9czkRLLlC7g89oJWqRRulK",
+  "org_3ABdwuK3qD7Saq590ZMQWY7AvVz",
+]);
  
 export const onRequest = clerkMiddleware(async (auth, context, next) => {
   const { isAuthenticated, userId, redirectToSignIn, has } = auth();
@@ -45,15 +48,26 @@ export const onRequest = clerkMiddleware(async (auth, context, next) => {
  
     try {
       const client = await clerkClient(context);
-      const memberships = await client.organizations.getOrganizationMembershipList({
+      const userOrgIds = await getUserOrgIds(context, userId);
-        organizationId: TARGET_ORG_ID,
+      const matchingOrgIds = userOrgIds.filter((id) => ADMIN_ORG_IDS.has(id));
-      });
  
-      const userMembership = memberships.data.find(
+      if (matchingOrgIds.length === 0) {
-        (m) => m.publicUserData?.userId === userId
+        return new Response(null, { status: 404 });
+      }
+ 
+      const membershipLists = await Promise.all(
+        matchingOrgIds.map((orgId) =>
+          client.organizations.getOrganizationMembershipList({ organizationId: orgId })
+        )
       );
  
-      if (!userMembership || userMembership.role !== "org:admin") {
+      const isAdmin = membershipLists.some((list) =>
+        list.data.some(
+          (m) => m.publicUserData?.userId === userId && m.role === "org:admin"
+        )
+      );
+ 
+      if (!isAdmin) {
         return new Response(null, { status: 404 });
       }
     } catch (e) {

src/pages/admin.astro

@@ -9,10 +9,182 @@ import Footer from '../components/Footer.astro';
   <NavBar slot="navbar">
     <NavItems slot="navItems" />
   </NavBar>
-  <div class="row mb-4" slot="page">
+  <div slot="page">
+    <div class="container my-4">
+      <div class="row mb-4">
         <div class="col-12">
           <h1>Admin Panel</h1>
+
+          <div class="accordion" id="adminAccordion">
+          <div class="accordion-item">
+            <h2 class="accordion-header" id="reindexHeading">
+              <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse"
+                      data-bs-target="#reindexCollapse" aria-expanded="false" aria-controls="reindexCollapse">
+                Reindex
+              </button>
+            </h2>
+            <div id="reindexCollapse" class="accordion-collapse collapse" aria-labelledby="reindexHeading"
+                data-bs-parent="#adminAccordion">
+              <div class="accordion-body">
+                <form id="reindexForm">
+                  <div class="mb-3">
+                    <div class="form-text mb-2">Select collections to reindex:</div>
+                    <div class="form-check">
+                      <input class="form-check-input" type="checkbox" id="reindexCards" name="cards" checked />
+                      <label class="form-check-label" for="reindexCards">Cards</label>
+                    </div>
+                    <div class="form-check">
+                      <input class="form-check-input" type="checkbox" id="reindexSkus" name="skus" checked />
+                      <label class="form-check-label" for="reindexSkus">SKUs</label>
+                    </div>
+                    <div class="form-check">
+                      <input class="form-check-input" type="checkbox" id="reindexInventory" name="inventory" checked />
+                      <label class="form-check-label" for="reindexInventory">Inventory</label>
                     </div>
                   </div>
+                  <div class="mb-3 form-check">
+                    <input class="form-check-input" type="checkbox" id="reindexRecreate" name="recreate" />
+                    <label class="form-check-label" for="reindexRecreate">
+                      Recreate index (drops and recreates collections; otherwise updates in place)
+                    </label>
+                  </div>
+                  <button type="submit" class="btn btn-purple" id="reindexRun">Run Reindex</button>
+                </form>
+              </div>
+            </div>
+          </div>
+        </div>
+        </div>
+      </div>
+    </div>
+
+    <!-- Reusable scrollable progress modal. Open via window.AdminProgress.open(title). -->
+    <div class="modal fade" id="adminProgressModal" tabindex="-1" aria-labelledby="adminProgressLabel" aria-hidden="true"
+         data-bs-backdrop="static" data-bs-keyboard="false">
+    <div class="modal-dialog modal-lg modal-dialog-scrollable">
+      <div class="modal-content">
+        <div class="modal-header">
+          <h5 class="modal-title" id="adminProgressLabel">Progress</h5>
+          <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"
+                  id="adminProgressClose" disabled></button>
+        </div>
+        <div class="modal-body p-0">
+          <pre id="adminProgressLog"
+               class="m-0 p-3 small"
+               style="max-height: 60vh; overflow-y: auto; white-space: pre-wrap; word-break: break-word;"></pre>
+        </div>
+        <div class="modal-footer">
+          <span class="me-auto small text-secondary" id="adminProgressStatus">Idle</span>
+          <button type="button" class="btn btn-secondary" data-bs-dismiss="modal"
+                  id="adminProgressDismiss" disabled>Close</button>
+        </div>
+        </div>
+      </div>
+    </div>
+  </div>
+
   <Footer slot="footer" />
 </Layout>
+
+<script>
+  // Reusable progress modal. Other admin features can call window.AdminProgress.
+  type ProgressHandle = {
+    append: (line: string) => void;
+    setStatus: (text: string) => void;
+    done: (text?: string) => void;
+  };
+
+  declare global {
+    interface Window {
+      AdminProgress: {
+        open: (title: string) => Promise<ProgressHandle>;
+      };
+    }
+  }
+
+  const getBootstrap = (): any => (window as any).bootstrap;
+
+  const openProgress = async (title: string): Promise<ProgressHandle> => {
+    const modalEl = document.getElementById('adminProgressModal')!;
+    const labelEl = document.getElementById('adminProgressLabel')!;
+    const logEl = document.getElementById('adminProgressLog')!;
+    const statusEl = document.getElementById('adminProgressStatus')!;
+    const closeBtn = document.getElementById('adminProgressClose') as HTMLButtonElement;
+    const dismissBtn = document.getElementById('adminProgressDismiss') as HTMLButtonElement;
+
+    labelEl.textContent = title;
+    logEl.textContent = '';
+    statusEl.textContent = 'Running...';
+    closeBtn.disabled = true;
+    dismissBtn.disabled = true;
+
+    const modal = getBootstrap().Modal.getOrCreateInstance(modalEl);
+    modal.show();
+
+    return {
+      append: (line: string) => {
+        logEl.textContent += (logEl.textContent ? '\n' : '') + line;
+        logEl.scrollTop = logEl.scrollHeight;
+      },
+      setStatus: (text: string) => { statusEl.textContent = text; },
+      done: (text = 'Done') => {
+        statusEl.textContent = text;
+        closeBtn.disabled = false;
+        dismissBtn.disabled = false;
+      },
+    };
+  };
+
+  window.AdminProgress = { open: openProgress };
+
+  // Reindex form wiring
+  const form = document.getElementById('reindexForm') as HTMLFormElement | null;
+  if (form) {
+    form.addEventListener('submit', async (e) => {
+      e.preventDefault();
+      const runBtn = document.getElementById('reindexRun') as HTMLButtonElement;
+      const body = {
+        cards: (document.getElementById('reindexCards') as HTMLInputElement).checked,
+        skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked,
+        inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked,
+        recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked,
+      };
+
+      runBtn.disabled = true;
+      const progress = await window.AdminProgress.open('Reindex');
+
+      try {
+        const resp = await fetch('/api/reindex', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(body),
+        });
+
+        if (!resp.ok || !resp.body) {
+          progress.append(`Request failed: ${resp.status} ${resp.statusText}`);
+          progress.done('Failed');
+          return;
+        }
+
+        const reader = resp.body.getReader();
+        const decoder = new TextDecoder();
+        let buf = '';
+        while (true) {
+          const { value, done } = await reader.read();
+          if (done) break;
+          buf += decoder.decode(value, { stream: true });
+          const lines = buf.split('\n');
+          buf = lines.pop() ?? '';
+          for (const line of lines) progress.append(line);
+        }
+        if (buf) progress.append(buf);
+        progress.done('Done');
+      } catch (err: any) {
+        progress.append(`Error: ${err?.message || String(err)}`);
+        progress.done('Failed');
+      } finally {
+        runBtn.disabled = false;
+      }
+    });
+  }
+</script>

src/pages/api/reindex.ts

@@ -0,0 +1,71 @@
+import type { APIRoute } from 'astro';
+import { db } from '../../db/index';
+import * as Indexing from '../../../scripts/pokemon-helper';
+
+export const POST: APIRoute = async ({ request }) => {
+    const { cards, skus, inventory, recreate } = await request.json().catch(() => ({} as any));
+
+    const encoder = new TextEncoder();
+    const stream = new ReadableStream({
+        async start(controller) {
+            const log = (msg: string) => {
+                controller.enqueue(encoder.encode(msg + '\n'));
+            };
+
+            try {
+                if (!cards && !skus && !inventory) {
+                    log('No collections selected.');
+                    return;
+                }
+
+                if (recreate) {
+                    if (cards) {
+                        log('Recreating "cards" collection...');
+                        await Indexing.createCardCollection(log);
+                    }
+                    if (skus) {
+                        log('Recreating "skus" collection...');
+                        await Indexing.createSkuCollection(log);
+                    }
+                    if (inventory) {
+                        log('Recreating "inventories" collection...');
+                        await Indexing.createInventoryCollection(log);
+                    }
+                }
+
+                if (cards) {
+                    log('Indexing "cards"...');
+                    await Indexing.upsertCardCollection(db, log);
+                }
+                if (skus) {
+                    log('Indexing "skus"...');
+                    await Indexing.upsertSkuCollection(db, log);
+                }
+                if (inventory) {
+                    log('Indexing "inventories"...');
+                    await Indexing.upsertInventoryCollection(db, log);
+                }
+
+                log('Reindex complete.');
+            } catch (e: any) {
+                const cause = e?.cause;
+                const causeMsg = cause?.message || (cause ? String(cause) : '');
+                const causeDetail = cause?.detail ? ` | detail: ${cause.detail}` : '';
+                const causeCode = cause?.code ? ` | code: ${cause.code}` : '';
+                log(`Error: ${e?.message || String(e)}`);
+                if (causeMsg) log(`Caused by: ${causeMsg}${causeCode}${causeDetail}`);
+                console.error('Reindex error:', e);
+            } finally {
+                controller.close();
+            }
+        },
+    });
+
+    return new Response(stream, {
+        headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+            'Cache-Control': 'no-cache, no-transform',
+            'X-Accel-Buffering': 'no',
+        },
+    });
+};