replace productId with cardId in price history chart plotting

update ebay link to have variant included in query
[feat] retry preload cards 3x
2026-06-04 15:18:27 -04:00 · 2026-06-04 14:20:39 -04:00 · 2026-05-29 22:25:09 -04:00 · 2026-05-29 16:29:02 -04:00 · 2026-05-28 21:15:29 -04:00 · 2026-05-28 16:12:48 -04:00
13 changed files with 956 additions and 642 deletions
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -20,7 +20,7 @@
    "chart.js": "^4.5.1",
    "csv": "^6.4.1",
    "dotenv": "^17.2.4",
-    "drizzle-orm": "^1.0.0-beta.15-859cf75",
+    "drizzle-orm": "1.0.0-beta.15-859cf75",
    "pg": "^8.20.0",
    "sass": "^1.97.3",
    "typesense": "^3.0.1"
@@ -29,7 +29,7 @@
    "@types/bootstrap": "^5.2.10",
    "@types/node": "^25.2.1",
    "@types/pg": "^8.18.0",
-    "drizzle-kit": "^1.0.0-beta.15-859cf75",
+    "drizzle-kit": "1.0.0-beta.15-859cf75",
    "typescript": "^5.9.3"
  }
 }
--- a/scripts/pokemon-helper.ts
+++ b/scripts/pokemon-helper.ts
@@ -12,6 +12,9 @@ const DollarToInt = (dollar: any) => {
    return Math.round(dollar * 100);
 }

+export type Logger = (msg: string) => void;
+const defaultLogger: Logger = (msg) => console.log(chalk.green(msg));
+


 export const Sleep = (ms: number) => {
@@ -39,7 +42,7 @@ export const GetNumberOrNull = (value: any): number | null => {


 // Delete and recreate the 'cards' index
-export const createCardCollection = async () => {
+export const createCardCollection = async (log: Logger = defaultLogger) => {
    try {
        await client.collections('cards').delete();
    } catch (error) {
@@ -59,7 +62,8 @@ export const createCardCollection = async () => {
            { name: 'setCode', type: 'string' },
            { name: 'cardType', type: 'string', facet: true },
            { name: 'energyType', type: 'string', facet: true },
-            { name: 'number', type: 'string', sort: true },
+            { name: 'number', type: 'string' },
+            { name: 'inumber', type: 'int32', optional: true, sort: true },
            { name: 'Artist', type: 'string' },
            { name: 'sealed', type: 'bool' },
            { name: 'releaseDate', type: 'int32' },
@@ -68,11 +72,11 @@ export const createCardCollection = async () => {
            // { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true }
        ],
    });
-    console.log(chalk.green('Collection "cards" created successfully.'));
+    log('Collection "cards" created successfully.');
 }

 // Delete and recreate the 'skus' index
-export const createSkuCollection = async () => {
+export const createSkuCollection = async (log: Logger = defaultLogger) => {
    try {
        await client.collections('skus').delete();
    } catch (error) {
@@ -89,11 +93,11 @@ export const createSkuCollection = async () => {
            { name: 'card_id', type: 'string', reference: 'cards.id', async_reference: true },
        ]
    });
-    console.log(chalk.green('Collection "skus" created successfully.'));
+    log('Collection "skus" created successfully.');
 }

 // Delete and recreate the 'inventory' index
-export const createInventoryCollection = async () => {
+export const createInventoryCollection = async (log: Logger = defaultLogger) => {
    try {
        await client.collections('inventories').delete();
    } catch (error) {
@@ -117,11 +121,11 @@ export const createInventoryCollection = async () => {
            { name: 'cardType', type: 'string' },
        ]
    });
-    console.log(chalk.green('Collection "inventories" created successfully.'));
+    log('Collection "inventories" created successfully.');
 }


-export const upsertCardCollection = async (db:DBInstance) => {
+export const upsertCardCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
    const pokemon = await db.query.cards.findMany({
        with: { set: true, tcgdata: true, prices: true },
    });
@@ -149,6 +153,7 @@ export const upsertCardCollection = async (db:DBInstance) => {
            cardType: card.cardType || "",
            energyType: card.energyType || "",
            number: card.number,
+            inumber: (card.number !== null) ? parseInt(card.number) : undefined,
            Artist: card.artist || "",
            sealed: card.sealed,
            content: [card.productName, card.productLineName, card.set?.setName || "", card.set?.setCode || "", card.number, card.rarityName, card.artist || ""].join(' '),
@@ -157,10 +162,10 @@ export const upsertCardCollection = async (db:DBInstance) => {
            // sku_id: card.prices.map(price => price.skuId.toString())
        };
    }), { action: 'upsert' });
-    console.log(chalk.green('Collection "cards" indexed successfully.'));
+    log('Collection "cards" indexed successfully.');
 }

-export const upsertSkuCollection = async (db:DBInstance) => {
+export const upsertSkuCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
    const skus = await db.query.skus.findMany();
    await client.collections('skus').documents().import(skus.map(sku => ({
        id: sku.skuId.toString(),
@@ -170,10 +175,10 @@ export const upsertSkuCollection = async (db:DBInstance) => {
        marketPrice: DollarToInt(sku.marketPrice),
        card_id: sku.cardId.toString(),
    })), { action: 'upsert' }); 
-    console.log(chalk.green('Collection "skus" indexed successfully.'));
+    log('Collection "skus" indexed successfully.');
 }

-export const upsertInventoryCollection = async (db:DBInstance) => {
+export const upsertInventoryCollection = async (db:DBInstance, log: Logger = defaultLogger) => {
    const inv = await db.query.inventory.findMany({
        with: { sku: { with: { card: { with: { set: true } } } } }
    });
@@ -198,25 +203,27 @@ export const upsertInventoryCollection = async (db:DBInstance) => {
            i.sku?.card?.artist || ""
        ].join(' '),
    })), { action: 'upsert' });
-    console.log(chalk.green('Collection "inventories" indexed successfully.'));
+    log('Collection "inventories" indexed successfully.');
 }





-export const UpdateVariants = async (db:DBInstance) => {
+export const UpdateVariants = async (db:DBInstance, log: Logger = (m) => console.log(m)) => {
    const updates = await db.execute(sql`update cards as c
 set 
    product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, 
-    sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
+    sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, inumber = a.inumber, artist = a.artist
 from (
    select t.product_id, b.variant,
        coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
        coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
        coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
        coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
-        coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
+        coalesce(o.number, regexp_replace(t.number,'^0+','')) as number,
+        nullif(regexp_replace(regexp_replace(coalesce(o.number,t.number),'/.*',''),'[^0-9]','','g'),'')::integer as inumber,
+        coalesce(o.artist, t.artist) as artist
    from tcg_cards t
    join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
    left join tcg_overrides o on t.product_id = o.product_id
@@ -229,22 +236,23 @@ where c.product_id = a.product_id and c.variant = a.variant and
    c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
 )
 `);
-    console.log(`Updated ${updates.rowCount} rows in cards table`);
+    log(`Updated ${updates.rowCount} rows in cards table`);

-    const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
+    const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", inumber, artist)
 select t.product_id, b.variant, 
 coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
 coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name, 
 coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type, 
-coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
+coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, regexp_replace(t.number,'^0+','')) as number, 
+nullif(regexp_replace(regexp_replace(coalesce(o.number,t.number),'/.*',''),'[^0-9]','','g'),'')::integer as inumber, coalesce(o.artist, t.artist) as artist
 from tcg_cards t
 join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
 left join tcg_overrides o on t.product_id = o.product_id
 where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
 `);
-    console.log(`Inserted ${inserts.rowCount} rows into cards table`);
+    log(`Inserted ${inserts.rowCount} rows into cards table`);

    const skuUpdates = await db.execute(sql`update skus s set card_id = c.card_id from cards c where s.product_id = c.product_id and s.variant = c.variant and s.card_id is distinct from c.card_id`);
-    console.log(`Updated ${skuUpdates.rowCount} rows in skus table`);
+    log(`Updated ${skuUpdates.rowCount} rows in skus table`);

 }
--- a/scripts/preload-tcgplayer.ts
+++ b/scripts/preload-tcgplayer.ts
@@ -1,106 +1,86 @@
 import 'dotenv/config';
 import * as schema from '../src/db/schema.ts';
-import { db, ClosePool } from '../src/db/index.ts';
+import { db, ClosePool, type DBInstance } from '../src/db/index.ts';

 import fs from "node:fs/promises";
 import path from "node:path";
-import chalk from 'chalk';
+import { pathToFileURL } from 'node:url';
 import * as helper from './pokemon-helper.ts';
-//import util from 'util';
+
+export type Logger = (msg: string) => void;
+const consoleLogger: Logger = (m) => console.log(m);
+
+export type RunImportOptions = {
+  sets?: string[];
+  log?: Logger;
+  runUpdateVariants?: boolean;
+  runCardUpsert?: boolean;
+};


-async function syncTcgplayer(cardSets:string[] = []) {
-
-  const productLines = [ "pokemon", "pokemon-japan" ];
-
-  // work from the available sets within the product line
-  for (const productLine of productLines) {
-    const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
-
-    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
-      method: 'POST',
-      headers: {'Content-Type': 'application/json',},
-      body: JSON.stringify(d),
-    });
-    if (!response.ok) {
-      console.error('Error notifying sync completion:', response.statusText);
-      process.exit(1);
-    }
-    const data = await response.json();
-
-    const setNames = data.results[0].aggregations.setName;
-    for (const setName of setNames) {
-      let processSet = true;
-      if (cardSets.length > 0) {
-        processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
-      }
-      if (processSet) {
-        console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
-        await syncProductLine(productLine, "setName", setName.urlValue);
-      }
-    }
-
-  }
-
-  console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
-}
-
-
-async function syncProductLine(productLine: string, field: string, fieldValue: string) {
+const syncProductLine = async (
+  database: DBInstance,
+  productLine: string,
+  field: string,
+  fieldValue: string,
+  allProductIds: Set<number>,
+  log: Logger,
+) => {
  let start = 0;
-  let size = 50;
+  const size = 50;
  let total = 1000000;

  while (start < total) {
-    console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
+    log(` Fetching items ${start} to ${start + size} of ${total}...`);

    const d = {
-          "algorithm":"sales_dismax",
-          "from":start,
-          "size":size,
-          "filters":{
-              "term":{"productLineName":[productLine], [field]:[fieldValue]} ,
-              "range":{},
-              "match":{}
-          },
-          "listingSearch":{
-              "context":{"cart":{}},
-              "filters":{"term":{
-                  "sellerStatus":"Live",
-                  "channelId":0
-                  },
-                  "range":{
-                      "quantity":{"gte":1}
-                  },
-                  "exclude":{"channelExclusion":0}
-              }
-          },
-          "context":{
-              "cart":{},
-              "shippingCountry":"US",
-              "userProfile":{}
-          },
-          "settings":{
-              "useFuzzySearch":false,
-              "didYouMean":{}
-          },
-          "sort":{}
-      };
-
-    //console.log(util.inspect(d, { depth: null }));
-    //process.exit(1);
-
-    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
+      "algorithm": "sales_dismax",
+      "from": start,
+      "size": size,
+      "filters": {
+        "term": { "productLineName": [productLine], [field]: [fieldValue] },
+        "range": {},
+        "match": {},
      },
-      body: JSON.stringify(d),
-    });
+      "listingSearch": {
+        "context": { "cart": {} },
+        "filters": {
+          "term": { "sellerStatus": "Live", "channelId": 0 },
+          "range": { "quantity": { "gte": 1 } },
+          "exclude": { "channelExclusion": 0 },
+        },
+      },
+      "context": { "cart": {}, "shippingCountry": "US", "userProfile": {} },
+      "settings": { "useFuzzySearch": false, "didYouMean": {} },
+      "sort": {},
+    };

-    if (!response.ok) {
-      console.error('Error notifying sync completion:', response.statusText);
-      process.exit(1);
+    let response: Response | null = null;
+    let lastError: unknown = null;
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      try {
+        const r = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(d),
+        });
+        if (r.ok) {
+          response = r;
+          break;
+        }
+        lastError = new Error(`TCGPlayer search request failed: ${r.status} ${r.statusText}`);
+      } catch (err) {
+        lastError = err;
+      }
+      if (attempt < 3) {
+        log(`   retry ${attempt}/2 for search request in 5s...`);
+        await helper.Sleep(5000);
+      }
+    }
+    if (!response) {
+      throw lastError instanceof Error
+        ? lastError
+        : new Error(`TCGPlayer search request failed: ${String(lastError)}`);
    }

    const data = await response.json();
@@ -108,28 +88,43 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s

    for (const item of data.results[0].results) {

-      // Check if productId already exists and skip if it does (to avoid hitting the API too much)
      if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
        continue;
      }

-      console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
+      log(` - ${item.productName} (ID: ${item.productId})`);

-      // Get product detail
-      const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
-        method: 'GET',
-      });
-      if (!detailResponse.ok) {
-        console.error('Error fetching product details:', detailResponse.statusText);
-        process.exit(1);
+      let detailResponse: Response | null = null;
+      let lastError: unknown = null;
+      for (let attempt = 1; attempt <= 3; attempt++) {
+        try {
+          const r = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
+            method: 'GET',
+          });
+          if (r.ok) {
+            detailResponse = r;
+            break;
+          }
+          lastError = new Error(`Error fetching product details for ${item.productId}: ${r.statusText}`);
+        } catch (err) {
+          lastError = err;
+        }
+        if (attempt < 3) {
+          log(`   retry ${attempt}/2 for product ${item.productId} in 5s...`);
+          await helper.Sleep(5000);
+        }
+      }
+      if (!detailResponse) {
+        throw lastError instanceof Error
+          ? lastError
+          : new Error(`Error fetching product details for ${item.productId}: ${String(lastError)}`);
      }
      const detailData = await detailResponse.json();


-      await db.insert(schema.tcgcards).values({
+      await database.insert(schema.tcgcards).values({
        productId: item.productId,
        productName: detailData.productName,
-        //productName: cleanProductName(item.productName),
        rarityName: item.rarityName,
        productLineName: detailData.productLineName,
        productLineUrlName: detailData.productLineUrlName,
@@ -167,7 +162,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
        target: schema.tcgcards.productId,
        set: {
          productName: detailData.productName,
-          //productName: cleanProductName(item.productName),
          rarityName: item.rarityName,
          productLineName: detailData.productLineName,
          productLineUrlName: detailData.productLineUrlName,
@@ -204,11 +198,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
        },
      });

-      // console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
-      // console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
-      // console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
-      // set is...
-      await db.insert(schema.sets).values({
+      await database.insert(schema.sets).values({
        setId: detailData.setId,
        setCode: detailData.setCode,
        setName: detailData.setName,
@@ -222,10 +212,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
        },
      });

-      // skus are...
      for (const skuItem of detailData.skus) {
-
-        await db.insert(schema.skus).values({
+        await database.insert(schema.skus).values({
          skuId: skuItem.sku,
          productId: detailData.productId,
          condition: skuItem.condition,
@@ -241,7 +229,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
        });
      }

-      // get image if it doesn't already exist
      const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
      if (!await helper.FileExists(imagePath)) {
        const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
@@ -249,37 +236,88 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
          const buffer = await imageResponse.arrayBuffer();
          await fs.writeFile(imagePath, Buffer.from(buffer));
        } else {
-          console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
+          log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`);
          await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
        }
      }

-      // be nice to the API and not send too many requests in a short time
      await helper.Sleep(300);
-
    }

    start += size;
  }
+};
+
+
+const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set<number>, log: Logger) => {
+  const productLines = ["pokemon", "pokemon-japan"];
+
+  for (const productLine of productLines) {
+    const d = {
+      "algorithm": "sales_dismax", "from": 0, "size": 1,
+      "filters": { "term": { "productLineName": [productLine] } },
+      "settings": { "useFuzzySearch": false },
+    };
+
+    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(d),
+    });
+    if (!response.ok) {
+      throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`);
+    }
+    const data = await response.json();
+
+    const setNames = data.results[0].aggregations.setName;
+    for (const setName of setNames) {
+      let processSet = true;
+      if (cardSets.length > 0) {
+        processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
+      }
+      if (processSet) {
+        log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`);
+        await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log);
+      }
+    }
+  }
+
+  log('All TCGPlayer data synchronized successfully!');
+};
+
+
+export const runImport = async (opts: RunImportOptions = {}) => {
+  const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts;
+
+  await fs.rm('missing_images.log', { force: true });
+
+  // When no set filter is provided, skip productIds already in the cards table
+  // (matches the CLI script's "no args" behavior).
+  const allProductIds = sets.length === 0
+    ? new Set<number>(
+        await db.select({ productId: schema.cards.productId }).from(schema.cards)
+          .then(rows => rows.map(row => row.productId))
+      )
+    : new Set<number>();
+
+  await syncTcgplayer(db, sets, allProductIds, log);
+
+  if (runUpdateVariants) {
+    log('Updating card variants...');
+    await helper.UpdateVariants(db, log);
+  }
+
+  if (runCardUpsert) {
+    log('Reindexing "cards" collection...');
+    await helper.upsertCardCollection(db, log);
+  }
+};
+
+
+// CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage.
+const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isCli) {
+  const args = process.argv.slice(2);
+  await runImport({ sets: args });
+  await ClosePool();
 }
-
-// clear the log file
-await fs.rm('missing_images.log', { force: true });
-let allProductIds = new Set();
-
-const args = process.argv.slice(2);
-if (args.length === 0) {
-  allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
-  await syncTcgplayer();
-}
-else {
-  await syncTcgplayer(args);
-}
-
-// update the card table with new/updated variants
-await helper.UpdateVariants(db);
-
-// index the card updates
-await helper.upsertCardCollection(db);
-
-await ClosePool();
--- a/src/assets/css/_bootstrap.scss
+++ b/src/assets/css/_bootstrap.scss
@@ -18,7 +18,7 @@
@import 'bootstrap/scss/containers';
@import 'bootstrap/scss/images';
@import 'bootstrap/scss/nav';
-// @import 'bootstrap/scss/accordion';
+@import 'bootstrap/scss/accordion';
@import 'bootstrap/scss/alert';
@import 'bootstrap/scss/badge';
 // @import 'bootstrap/scss/breadcrumb';
--- a/src/db/schema.ts
+++ b/src/db/schema.ts
@@ -56,6 +56,7 @@ export const cards = pokeSchema.table('cards', {
 	cardType: varchar({ length: 100 }),
 	energyType: varchar({ length: 100 }),
 	number: varchar({ length: 50 }),
+	inumber: integer(),
 	artist: varchar({ length: 255 }),
 },
 (table) => [
--- a/src/middleware.ts
+++ b/src/middleware.ts
@@ -1,5 +1,4 @@
 import { clerkMiddleware, createRouteMatcher, clerkClient } from '@clerk/astro/server';
-import type { MiddlewareNext } from 'astro';
 import 'dotenv/config';
 
 declare global {
@@ -11,9 +10,13 @@ declare global {
 }
 
 const isProtectedRoute = createRouteMatcher(['/pokemon']);
-const isAdminRoute = createRouteMatcher(['/admin']);
+const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex', '/api/preload-tcgplayer']);

 const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK";
+const ADMIN_ORG_IDS = new Set([
+  "org_3Baav9czkRLLlC7g89oJWqRRulK",
+  "org_3ABdwuK3qD7Saq590ZMQWY7AvVz",
+]);
 
 export const onRequest = clerkMiddleware(async (auth, context, next) => {
  const { isAuthenticated, userId, redirectToSignIn, has } = auth();
@@ -45,15 +48,26 @@ export const onRequest = clerkMiddleware(async (auth, context, next) => {
 
    try {
      const client = await clerkClient(context);
-      const memberships = await client.organizations.getOrganizationMembershipList({
-        organizationId: TARGET_ORG_ID,
-      });
+      const userOrgIds = await getUserOrgIds(context, userId);
+      const matchingOrgIds = userOrgIds.filter((id) => ADMIN_ORG_IDS.has(id));
 
-      const userMembership = memberships.data.find(
-        (m) => m.publicUserData?.userId === userId
+      if (matchingOrgIds.length === 0) {
+        return new Response(null, { status: 404 });
+      }
+ 
+      const membershipLists = await Promise.all(
+        matchingOrgIds.map((orgId) =>
+          client.organizations.getOrganizationMembershipList({ organizationId: orgId })
+        )
      );
 
-      if (!userMembership || userMembership.role !== "org:admin") {
+      const isAdmin = membershipLists.some((list) =>
+        list.data.some(
+          (m) => m.publicUserData?.userId === userId && m.role === "org:admin"
+        )
+      );
+ 
+      if (!isAdmin) {
        return new Response(null, { status: 404 });
      }
    } catch (e) {
--- a/src/pages/admin.astro
+++ b/src/pages/admin.astro
@@ -9,10 +9,217 @@ import Footer from '../components/Footer.astro';
  <NavBar slot="navbar">
    <NavItems slot="navItems" />
  </NavBar>
-  <div class="row mb-4" slot="page">
-    <div class="col-12">
-      <h1>Admin Panel</h1>
+  <div slot="page">
+    <div class="container my-4">
+      <div class="row mb-4">
+        <div class="col-12">
+          <h1>Admin Panel</h1>
+
+          <div class="accordion" id="adminAccordion">
+          <div class="accordion-item">
+            <h2 class="accordion-header" id="reindexHeading">
+              <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse"
+                      data-bs-target="#reindexCollapse" aria-expanded="false" aria-controls="reindexCollapse">
+                Reindex
+              </button>
+            </h2>
+            <div id="reindexCollapse" class="accordion-collapse collapse" aria-labelledby="reindexHeading"
+                data-bs-parent="#adminAccordion">
+              <div class="accordion-body">
+                <form id="reindexForm">
+                  <div class="mb-3">
+                    <div class="form-text mb-2">Select collections to reindex:</div>
+                    <div class="form-check">
+                      <input class="form-check-input" type="checkbox" id="reindexCards" name="cards" checked />
+                      <label class="form-check-label" for="reindexCards">Cards</label>
+                    </div>
+                    <div class="form-check">
+                      <input class="form-check-input" type="checkbox" id="reindexSkus" name="skus" checked />
+                      <label class="form-check-label" for="reindexSkus">SKUs</label>
+                    </div>
+                    <div class="form-check">
+                      <input class="form-check-input" type="checkbox" id="reindexInventory" name="inventory" checked />
+                      <label class="form-check-label" for="reindexInventory">Inventory</label>
+                    </div>
+                  </div>
+                  <div class="mb-3 form-check">
+                    <input class="form-check-input" type="checkbox" id="reindexRecreate" name="recreate" />
+                    <label class="form-check-label" for="reindexRecreate">
+                      Recreate index (drops and recreates collections; otherwise updates in place)
+                    </label>
+                  </div>
+                  <button type="submit" class="btn btn-purple" id="reindexRun">Run Reindex</button>
+                </form>
+              </div>
+            </div>
+          </div>
+
+          <div class="accordion-item">
+            <h2 class="accordion-header" id="tcgImportHeading">
+              <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse"
+                      data-bs-target="#tcgImportCollapse" aria-expanded="false" aria-controls="tcgImportCollapse">
+                TCG Player Import
+              </button>
+            </h2>
+            <div id="tcgImportCollapse" class="accordion-collapse collapse" aria-labelledby="tcgImportHeading"
+                data-bs-parent="#adminAccordion">
+              <div class="accordion-body">
+                <form id="tcgImportForm">
+                  <div class="mb-3">
+                    <label for="tcgImportSetName" class="form-label">Set Name</label>
+                    <input type="text" class="form-control" id="tcgImportSetName" name="setName"
+                           placeholder="e.g. Surging Sparks" autocomplete="off" required />
+                    <div class="form-text">Matches any set whose name contains this text (case-insensitive).</div>
+                  </div>
+                  <button type="submit" class="btn btn-purple" id="tcgImportRun">Run Import</button>
+                </form>
+              </div>
+            </div>
+          </div>
+        </div>
+        </div>
+      </div>
+    </div>
+
+    <!-- Reusable scrollable progress modal. Open via window.AdminProgress.open(title). -->
+    <div class="modal fade" id="adminProgressModal" tabindex="-1" aria-labelledby="adminProgressLabel" aria-hidden="true"
+         data-bs-backdrop="static" data-bs-keyboard="false">
+    <div class="modal-dialog modal-lg modal-dialog-scrollable">
+      <div class="modal-content">
+        <div class="modal-header">
+          <h5 class="modal-title" id="adminProgressLabel">Progress</h5>
+          <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"
+                  id="adminProgressClose" disabled></button>
+        </div>
+        <div class="modal-body p-0">
+          <pre id="adminProgressLog"
+               class="m-0 p-3 small"
+               style="max-height: 60vh; overflow-y: auto; white-space: pre-wrap; word-break: break-word;"></pre>
+        </div>
+        <div class="modal-footer">
+          <span class="me-auto small text-secondary" id="adminProgressStatus">Idle</span>
+          <button type="button" class="btn btn-secondary" data-bs-dismiss="modal"
+                  id="adminProgressDismiss" disabled>Close</button>
+        </div>
+        </div>
+      </div>
    </div>
  </div>
+
  <Footer slot="footer" />
 </Layout>
+
+<script>
+  // Reusable progress modal. Other admin features can call window.AdminProgress.
+  type ProgressHandle = {
+    append: (line: string) => void;
+    setStatus: (text: string) => void;
+    done: (text?: string) => void;
+  };
+
+  declare global {
+    interface Window {
+      AdminProgress: {
+        open: (title: string) => Promise<ProgressHandle>;
+      };
+    }
+  }
+
+  const getBootstrap = (): any => (window as any).bootstrap;
+
+  const openProgress = async (title: string): Promise<ProgressHandle> => {
+    const modalEl = document.getElementById('adminProgressModal')!;
+    const labelEl = document.getElementById('adminProgressLabel')!;
+    const logEl = document.getElementById('adminProgressLog')!;
+    const statusEl = document.getElementById('adminProgressStatus')!;
+    const closeBtn = document.getElementById('adminProgressClose') as HTMLButtonElement;
+    const dismissBtn = document.getElementById('adminProgressDismiss') as HTMLButtonElement;
+
+    labelEl.textContent = title;
+    logEl.textContent = '';
+    statusEl.textContent = 'Running...';
+    closeBtn.disabled = true;
+    dismissBtn.disabled = true;
+
+    const modal = getBootstrap().Modal.getOrCreateInstance(modalEl);
+    modal.show();
+
+    return {
+      append: (line: string) => {
+        logEl.textContent += (logEl.textContent ? '\n' : '') + line;
+        logEl.scrollTop = logEl.scrollHeight;
+      },
+      setStatus: (text: string) => { statusEl.textContent = text; },
+      done: (text = 'Done') => {
+        statusEl.textContent = text;
+        closeBtn.disabled = false;
+        dismissBtn.disabled = false;
+      },
+    };
+  };
+
+  window.AdminProgress = { open: openProgress };
+
+  // Stream a POST JSON request line-by-line into a progress modal.
+  const streamToProgress = async (url: string, body: unknown, title: string, runBtn: HTMLButtonElement) => {
+    runBtn.disabled = true;
+    const progress = await window.AdminProgress.open(title);
+    try {
+      const resp = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(body),
+      });
+      if (!resp.ok || !resp.body) {
+        progress.append(`Request failed: ${resp.status} ${resp.statusText}`);
+        progress.done('Failed');
+        return;
+      }
+      const reader = resp.body.getReader();
+      const decoder = new TextDecoder();
+      let buf = '';
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        buf += decoder.decode(value, { stream: true });
+        const lines = buf.split('\n');
+        buf = lines.pop() ?? '';
+        for (const line of lines) progress.append(line);
+      }
+      if (buf) progress.append(buf);
+      progress.done('Done');
+    } catch (err: any) {
+      progress.append(`Error: ${err?.message || String(err)}`);
+      progress.done('Failed');
+    } finally {
+      runBtn.disabled = false;
+    }
+  };
+
+  // Reindex form wiring
+  const reindexForm = document.getElementById('reindexForm') as HTMLFormElement | null;
+  if (reindexForm) {
+    reindexForm.addEventListener('submit', (e) => {
+      e.preventDefault();
+      const body = {
+        cards: (document.getElementById('reindexCards') as HTMLInputElement).checked,
+        skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked,
+        inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked,
+        recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked,
+      };
+      streamToProgress('/api/reindex', body, 'Reindex',
+        document.getElementById('reindexRun') as HTMLButtonElement);
+    });
+  }
+
+  // TCG Player import form wiring
+  const tcgImportForm = document.getElementById('tcgImportForm') as HTMLFormElement | null;
+  if (tcgImportForm) {
+    tcgImportForm.addEventListener('submit', (e) => {
+      e.preventDefault();
+      const setName = (document.getElementById('tcgImportSetName') as HTMLInputElement).value.trim();
+      streamToProgress('/api/preload-tcgplayer', { setName }, `TCG Player Import: ${setName || '(none)'}`,
+        document.getElementById('tcgImportRun') as HTMLButtonElement);
+    });
+  }
+</script>
--- a/src/pages/api/preload-tcgplayer.ts
+++ b/src/pages/api/preload-tcgplayer.ts
@@ -0,0 +1,43 @@
+import type { APIRoute } from 'astro';
+import { runImport } from '../../../scripts/preload-tcgplayer';
+
+export const POST: APIRoute = async ({ request }) => {
+    const { setName } = await request.json().catch(() => ({} as any));
+    const trimmed = typeof setName === 'string' ? setName.trim() : '';
+
+    const encoder = new TextEncoder();
+    const stream = new ReadableStream({
+        async start(controller) {
+            const log = (msg: string) => {
+                controller.enqueue(encoder.encode(msg + '\n'));
+            };
+
+            try {
+                if (!trimmed) {
+                    log('Set name is required.');
+                    return;
+                }
+
+                log(`Starting TCGPlayer import for set: "${trimmed}"`);
+                await runImport({ sets: [trimmed], log });
+                log('TCGPlayer import complete.');
+            } catch (e: any) {
+                const cause = e?.cause;
+                const causeMsg = cause?.message || (cause ? String(cause) : '');
+                log(`Error: ${e?.message || String(e)}`);
+                if (causeMsg) log(`Caused by: ${causeMsg}`);
+                console.error('TCGPlayer import error:', e);
+            } finally {
+                controller.close();
+            }
+        },
+    });
+
+    return new Response(stream, {
+        headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+            'Cache-Control': 'no-cache, no-transform',
+            'X-Accel-Buffering': 'no',
+        },
+    });
+};
--- a/src/pages/api/reindex.ts
+++ b/src/pages/api/reindex.ts
@@ -0,0 +1,71 @@
+import type { APIRoute } from 'astro';
+import { db } from '../../db/index';
+import * as Indexing from '../../../scripts/pokemon-helper';
+
+export const POST: APIRoute = async ({ request }) => {
+    const { cards, skus, inventory, recreate } = await request.json().catch(() => ({} as any));
+
+    const encoder = new TextEncoder();
+    const stream = new ReadableStream({
+        async start(controller) {
+            const log = (msg: string) => {
+                controller.enqueue(encoder.encode(msg + '\n'));
+            };
+
+            try {
+                if (!cards && !skus && !inventory) {
+                    log('No collections selected.');
+                    return;
+                }
+
+                if (recreate) {
+                    if (cards) {
+                        log('Recreating "cards" collection...');
+                        await Indexing.createCardCollection(log);
+                    }
+                    if (skus) {
+                        log('Recreating "skus" collection...');
+                        await Indexing.createSkuCollection(log);
+                    }
+                    if (inventory) {
+                        log('Recreating "inventories" collection...');
+                        await Indexing.createInventoryCollection(log);
+                    }
+                }
+
+                if (cards) {
+                    log('Indexing "cards"...');
+                    await Indexing.upsertCardCollection(db, log);
+                }
+                if (skus) {
+                    log('Indexing "skus"...');
+                    await Indexing.upsertSkuCollection(db, log);
+                }
+                if (inventory) {
+                    log('Indexing "inventories"...');
+                    await Indexing.upsertInventoryCollection(db, log);
+                }
+
+                log('Reindex complete.');
+            } catch (e: any) {
+                const cause = e?.cause;
+                const causeMsg = cause?.message || (cause ? String(cause) : '');
+                const causeDetail = cause?.detail ? ` | detail: ${cause.detail}` : '';
+                const causeCode = cause?.code ? ` | code: ${cause.code}` : '';
+                log(`Error: ${e?.message || String(e)}`);
+                if (causeMsg) log(`Caused by: ${causeMsg}${causeCode}${causeDetail}`);
+                console.error('Reindex error:', e);
+            } finally {
+                controller.close();
+            }
+        },
+    });
+
+    return new Response(stream, {
+        headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+            'Cache-Control': 'no-cache, no-transform',
+            'X-Accel-Buffering': 'no',
+        },
+    });
+};
--- a/src/pages/partials/card-modal.astro
+++ b/src/pages/partials/card-modal.astro
@@ -84,7 +84,7 @@ for (const price of card?.prices ?? []) {

 // ── Price history for chart ───────────────────────────────────────────────
 const cardSkus = card?.prices?.length
-  ? await db.select().from(skus).where(eq(skus.productId, card.productId))
+  ? await db.select().from(skus).where(eq(skus.cardId, card.cardId))
  : [];

 const skuIds = cardSkus.map(s => s.skuId);
@@ -151,7 +151,7 @@ for (const price of card?.prices ?? []) {
 const availableVariants = [...new Set(cardSkus.map(s => s.variant))].sort();

 const ebaySearchUrl = (card: any) => {
-  return `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(card?.productUrlName)}+${encodeURIComponent(card?.set?.setUrlName)}+${encodeURIComponent(card?.number)}&LH_Sold=1&Graded=No&_dcat=183454`;
+  return `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(card?.productUrlName)}+${encodeURIComponent(card?.set?.setUrlName)}+${encodeURIComponent(card?.number)}+${encodeURIComponent(card?.variant)}&LH_Sold=1&Graded=No&_dcat=183454`;
 };

 const altSearchUrl = (card: any) => {
@@ -547,6 +547,8 @@ const altSearchUrl = (card: any) => {
                  </div>
                </div>

+                
+
              </div>

              <!-- External links column -->
--- a/src/pages/partials/cards.astro
+++ b/src/pages/partials/cards.astro
@@ -22,14 +22,14 @@ const facetFields:any = {

 // ── Allowed sort values ───────────────────────────────────────────────────
 const sortMap: Record<string, string> = {
-    'releaseDate:desc,number:asc': '_text_match:asc,releaseDate:desc,number:asc',
-    'releaseDate:asc,number:asc':  '_text_match:asc,releaseDate:asc,number:asc',
-    'marketPrice:desc':            'marketPrice:desc,releaseDate:desc,number:asc',
-    'marketPrice:asc':             'marketPrice:asc,releaseDate:desc,number:asc',
-    'number:asc':                  '_text_match:asc,number:asc',
-    'number:desc':                 '_text_match:asc,number:desc',
+    'releaseDate:desc,number:asc': '_text_match:asc,releaseDate:desc,inumber(missing_values:last):asc',
+    'releaseDate:asc,number:asc':  '_text_match:asc,releaseDate:asc,inumber(missing_values:last):asc',
+    'marketPrice:desc':            'marketPrice:desc,releaseDate:desc,inumber(missing_values:last):asc',
+    'marketPrice:asc':             'marketPrice:asc,releaseDate:desc,inumber(missing_values:last):asc',
+    'number:asc':                  '_text_match:asc,inumber(missing_values:last):asc',
+    'number:desc':                 '_text_match:asc,inumber(missing_values:last):desc',
 };
-const DEFAULT_SORT = '_text_match:asc,releaseDate:desc,number:asc';
+const DEFAULT_SORT = '_text_match:asc,releaseDate:desc,inumber(missing_values:last):asc';

 // get the query from post request using form data
 const formData = await Astro.request.formData();
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,5 +1,5 @@
 {
  "extends": "astro/tsconfigs/strict",
-  "include": [".astro/types.d.ts", "src/**/*"],
+  "include": [".astro/types.d.ts", "src/**/*", "scripts/**/*"],
  "exclude": ["dist"]
 }
Author	SHA1	Message	Date
Zach Harding	c0055ed713	replace productId with cardId in price history chart plotting	2026-06-04 15:18:27 -04:00
Zach Harding	96af72c7f4	update ebay link to have variant included in query	2026-06-04 14:20:39 -04:00
Thad Miller	afae3f445e	[feat] retry preload cards 3x	2026-05-29 22:25:09 -04:00
Thad Miller	e7c71e1c75	[feat] rework card number to be int32 for sorting	2026-05-29 16:29:02 -04:00
Thad Miller	9afc600e63	[bugfix] remove leading zeros from number	2026-05-28 21:15:29 -04:00
Thad Miller	2cf47d2b15	[feat] tcg player import added to admin page	2026-05-28 16:12:48 -04:00
Thad Miller	b0dbe7ced5	[feat] reindex added to admin	2026-05-28 15:14:18 -04:00
Thad Miller	ae0f3d6683	[bugfix] allow local clerk to work	2026-05-28 14:42:50 -04:00