[feat] tcg player import added to admin page

scripts/pokemon-helper.ts

@@ -208,7 +208,7 @@ export const upsertInventoryCollection = async (db:DBInstance, log: Logger = def
 
 
 
-export const UpdateVariants = async (db:DBInstance) => {
+export const UpdateVariants = async (db:DBInstance, log: Logger = (m) => console.log(m)) => {
     const updates = await db.execute(sql`update cards as c
 set 
     product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, 
@@ -232,7 +232,7 @@ where c.product_id = a.product_id and c.variant = a.variant and
     c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
 )
 `);
-    console.log(`Updated ${updates.rowCount} rows in cards table`);
+    log(`Updated ${updates.rowCount} rows in cards table`);
 
     const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
 select t.product_id, b.variant, 
@@ -245,9 +245,9 @@ join (select distinct product_id, variant from skus) b on t.product_id = b.produ
 left join tcg_overrides o on t.product_id = o.product_id
 where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
 `);
-    console.log(`Inserted ${inserts.rowCount} rows into cards table`);
+    log(`Inserted ${inserts.rowCount} rows into cards table`);
 
     const skuUpdates = await db.execute(sql`update skus s set card_id = c.card_id from cards c where s.product_id = c.product_id and s.variant = c.variant and s.card_id is distinct from c.card_id`);
-    console.log(`Updated ${skuUpdates.rowCount} rows in skus table`);
+    log(`Updated ${skuUpdates.rowCount} rows in skus table`);
 
 }

scripts/preload-tcgplayer.ts

@@ -1,106 +1,68 @@
 import 'dotenv/config';
 import * as schema from '../src/db/schema.ts';
-import { db, ClosePool } from '../src/db/index.ts';
+import { db, ClosePool, type DBInstance } from '../src/db/index.ts';
 
 import fs from "node:fs/promises";
 import path from "node:path";
-import chalk from 'chalk';
+import { pathToFileURL } from 'node:url';
 import * as helper from './pokemon-helper.ts';
-//import util from 'util';
+
+export type Logger = (msg: string) => void;
+const consoleLogger: Logger = (m) => console.log(m);
+
+export type RunImportOptions = {
+  sets?: string[];
+  log?: Logger;
+  runUpdateVariants?: boolean;
+  runCardUpsert?: boolean;
+};
 
 
-async function syncTcgplayer(cardSets:string[] = []) {
+const syncProductLine = async (
-
+  database: DBInstance,
-  const productLines = [ "pokemon", "pokemon-japan" ];
+  productLine: string,
-
+  field: string,
-  // work from the available sets within the product line
+  fieldValue: string,
-  for (const productLine of productLines) {
+  allProductIds: Set<number>,
-    const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
+  log: Logger,
-
+) => {
-    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
-      method: 'POST',
-      headers: {'Content-Type': 'application/json',},
-      body: JSON.stringify(d),
-    });
-    if (!response.ok) {
-      console.error('Error notifying sync completion:', response.statusText);
-      process.exit(1);
-    }
-    const data = await response.json();
-
-    const setNames = data.results[0].aggregations.setName;
-    for (const setName of setNames) {
-      let processSet = true;
-      if (cardSets.length > 0) {
-        processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
-      }
-      if (processSet) {
-        console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
-        await syncProductLine(productLine, "setName", setName.urlValue);
-      }
-    }
-
-  }
-
-  console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
-}
-
-
-async function syncProductLine(productLine: string, field: string, fieldValue: string) {
   let start = 0;
-  let size = 50;
+  const size = 50;
   let total = 1000000;
 
   while (start < total) {
-    console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
+    log(` Fetching items ${start} to ${start + size} of ${total}...`);
 
     const d = {
-          "algorithm":"sales_dismax",
+      "algorithm": "sales_dismax",
-          "from":start,
+      "from": start,
-          "size":size,
+      "size": size,
-          "filters":{
+      "filters": {
-              "term":{"productLineName":[productLine], [field]:[fieldValue]} ,
+        "term": { "productLineName": [productLine], [field]: [fieldValue] },
-              "range":{},
+        "range": {},
-              "match":{}
+        "match": {},
-          },
+      },
-          "listingSearch":{
+      "listingSearch": {
-              "context":{"cart":{}},
+        "context": { "cart": {} },
-              "filters":{"term":{
+        "filters": {
-                  "sellerStatus":"Live",
+          "term": { "sellerStatus": "Live", "channelId": 0 },
-                  "channelId":0
+          "range": { "quantity": { "gte": 1 } },
-                  },
+          "exclude": { "channelExclusion": 0 },
-                  "range":{
+        },
-                      "quantity":{"gte":1}
+      },
-                  },
+      "context": { "cart": {}, "shippingCountry": "US", "userProfile": {} },
-                  "exclude":{"channelExclusion":0}
+      "settings": { "useFuzzySearch": false, "didYouMean": {} },
-              }
+      "sort": {},
-          },
+    };
-          "context":{
-              "cart":{},
-              "shippingCountry":"US",
-              "userProfile":{}
-          },
-          "settings":{
-              "useFuzzySearch":false,
-              "didYouMean":{}
-          },
-          "sort":{}
-      };
-
-    //console.log(util.inspect(d, { depth: null }));
-    //process.exit(1);
 
     const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
       method: 'POST',
-      headers: {
+      headers: { 'Content-Type': 'application/json' },
-        'Content-Type': 'application/json',
-      },
       body: JSON.stringify(d),
     });
 
     if (!response.ok) {
-      console.error('Error notifying sync completion:', response.statusText);
+      throw new Error(`TCGPlayer search request failed: ${response.status} ${response.statusText}`);
-      process.exit(1);
     }
 
     const data = await response.json();
@@ -108,28 +70,24 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
 
     for (const item of data.results[0].results) {
 
-      // Check if productId already exists and skip if it does (to avoid hitting the API too much)
       if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
         continue;
       }
 
-      console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
+      log(` - ${item.productName} (ID: ${item.productId})`);
 
-      // Get product detail
       const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
         method: 'GET',
       });
       if (!detailResponse.ok) {
-        console.error('Error fetching product details:', detailResponse.statusText);
+        throw new Error(`Error fetching product details for ${item.productId}: ${detailResponse.statusText}`);
-        process.exit(1);
       }
       const detailData = await detailResponse.json();
 
 
-      await db.insert(schema.tcgcards).values({
+      await database.insert(schema.tcgcards).values({
         productId: item.productId,
         productName: detailData.productName,
-        //productName: cleanProductName(item.productName),
         rarityName: item.rarityName,
         productLineName: detailData.productLineName,
         productLineUrlName: detailData.productLineUrlName,
@@ -167,7 +125,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         target: schema.tcgcards.productId,
         set: {
           productName: detailData.productName,
-          //productName: cleanProductName(item.productName),
           rarityName: item.rarityName,
           productLineName: detailData.productLineName,
           productLineUrlName: detailData.productLineUrlName,
@@ -204,11 +161,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         },
       });
 
-      // console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
+      await database.insert(schema.sets).values({
-      // console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
-      // console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
-      // set is...
-      await db.insert(schema.sets).values({
         setId: detailData.setId,
         setCode: detailData.setCode,
         setName: detailData.setName,
@@ -222,10 +175,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         },
       });
 
-      // skus are...
       for (const skuItem of detailData.skus) {
-
+        await database.insert(schema.skus).values({
-        await db.insert(schema.skus).values({
           skuId: skuItem.sku,
           productId: detailData.productId,
           condition: skuItem.condition,
@@ -241,7 +192,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         });
       }
 
-      // get image if it doesn't already exist
       const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
       if (!await helper.FileExists(imagePath)) {
         const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
@@ -249,37 +199,88 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
           const buffer = await imageResponse.arrayBuffer();
           await fs.writeFile(imagePath, Buffer.from(buffer));
         } else {
-          console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
+          log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`);
           await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
         }
       }
 
-      // be nice to the API and not send too many requests in a short time
       await helper.Sleep(300);
-
     }
 
     start += size;
   }
+};
+
+
+const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set<number>, log: Logger) => {
+  const productLines = ["pokemon", "pokemon-japan"];
+
+  for (const productLine of productLines) {
+    const d = {
+      "algorithm": "sales_dismax", "from": 0, "size": 1,
+      "filters": { "term": { "productLineName": [productLine] } },
+      "settings": { "useFuzzySearch": false },
+    };
+
+    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(d),
+    });
+    if (!response.ok) {
+      throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`);
+    }
+    const data = await response.json();
+
+    const setNames = data.results[0].aggregations.setName;
+    for (const setName of setNames) {
+      let processSet = true;
+      if (cardSets.length > 0) {
+        processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
+      }
+      if (processSet) {
+        log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`);
+        await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log);
+      }
+    }
+  }
+
+  log('All TCGPlayer data synchronized successfully!');
+};
+
+
+export const runImport = async (opts: RunImportOptions = {}) => {
+  const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts;
+
+  await fs.rm('missing_images.log', { force: true });
+
+  // When no set filter is provided, skip productIds already in the cards table
+  // (matches the CLI script's "no args" behavior).
+  const allProductIds = sets.length === 0
+    ? new Set<number>(
+        await db.select({ productId: schema.cards.productId }).from(schema.cards)
+          .then(rows => rows.map(row => row.productId))
+      )
+    : new Set<number>();
+
+  await syncTcgplayer(db, sets, allProductIds, log);
+
+  if (runUpdateVariants) {
+    log('Updating card variants...');
+    await helper.UpdateVariants(db, log);
+  }
+
+  if (runCardUpsert) {
+    log('Reindexing "cards" collection...');
+    await helper.upsertCardCollection(db, log);
+  }
+};
+
+
+// CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage.
+const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isCli) {
+  const args = process.argv.slice(2);
+  await runImport({ sets: args });
+  await ClosePool();
 }
-
-// clear the log file
-await fs.rm('missing_images.log', { force: true });
-let allProductIds = new Set();
-
-const args = process.argv.slice(2);
-if (args.length === 0) {
-  allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
-  await syncTcgplayer();
-}
-else {
-  await syncTcgplayer(args);
-}
-
-// update the card table with new/updated variants
-await helper.UpdateVariants(db);
-
-// index the card updates
-await helper.upsertCardCollection(db);
-
-await ClosePool();

src/middleware.ts

@@ -10,7 +10,7 @@ declare global {
 }
  
 const isProtectedRoute = createRouteMatcher(['/pokemon']);
-const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex']);
+const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex', '/api/preload-tcgplayer']);
 
 const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK";
 const ADMIN_ORG_IDS = new Set([

src/pages/admin.astro

@@ -53,6 +53,29 @@ import Footer from '../components/Footer.astro';
               </div>
             </div>
           </div>
+
+          <div class="accordion-item">
+            <h2 class="accordion-header" id="tcgImportHeading">
+              <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse"
+                      data-bs-target="#tcgImportCollapse" aria-expanded="false" aria-controls="tcgImportCollapse">
+                TCG Player Import
+              </button>
+            </h2>
+            <div id="tcgImportCollapse" class="accordion-collapse collapse" aria-labelledby="tcgImportHeading"
+                data-bs-parent="#adminAccordion">
+              <div class="accordion-body">
+                <form id="tcgImportForm">
+                  <div class="mb-3">
+                    <label for="tcgImportSetName" class="form-label">Set Name</label>
+                    <input type="text" class="form-control" id="tcgImportSetName" name="setName"
+                           placeholder="e.g. Surging Sparks" autocomplete="off" required />
+                    <div class="form-text">Matches any set whose name contains this text (case-insensitive).</div>
+                  </div>
+                  <button type="submit" class="btn btn-purple" id="tcgImportRun">Run Import</button>
+                </form>
+              </div>
+            </div>
+          </div>
         </div>
         </div>
       </div>
@@ -137,54 +160,66 @@ import Footer from '../components/Footer.astro';
 
   window.AdminProgress = { open: openProgress };
 
+  // Stream a POST JSON request line-by-line into a progress modal.
+  const streamToProgress = async (url: string, body: unknown, title: string, runBtn: HTMLButtonElement) => {
+    runBtn.disabled = true;
+    const progress = await window.AdminProgress.open(title);
+    try {
+      const resp = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(body),
+      });
+      if (!resp.ok || !resp.body) {
+        progress.append(`Request failed: ${resp.status} ${resp.statusText}`);
+        progress.done('Failed');
+        return;
+      }
+      const reader = resp.body.getReader();
+      const decoder = new TextDecoder();
+      let buf = '';
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        buf += decoder.decode(value, { stream: true });
+        const lines = buf.split('\n');
+        buf = lines.pop() ?? '';
+        for (const line of lines) progress.append(line);
+      }
+      if (buf) progress.append(buf);
+      progress.done('Done');
+    } catch (err: any) {
+      progress.append(`Error: ${err?.message || String(err)}`);
+      progress.done('Failed');
+    } finally {
+      runBtn.disabled = false;
+    }
+  };
+
   // Reindex form wiring
-  const form = document.getElementById('reindexForm') as HTMLFormElement | null;
+  const reindexForm = document.getElementById('reindexForm') as HTMLFormElement | null;
-  if (form) {
+  if (reindexForm) {
-    form.addEventListener('submit', async (e) => {
+    reindexForm.addEventListener('submit', (e) => {
       e.preventDefault();
-      const runBtn = document.getElementById('reindexRun') as HTMLButtonElement;
       const body = {
         cards: (document.getElementById('reindexCards') as HTMLInputElement).checked,
         skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked,
         inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked,
         recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked,
       };
+      streamToProgress('/api/reindex', body, 'Reindex',
+        document.getElementById('reindexRun') as HTMLButtonElement);
+    });
+  }
 
-      runBtn.disabled = true;
+  // TCG Player import form wiring
-      const progress = await window.AdminProgress.open('Reindex');
+  const tcgImportForm = document.getElementById('tcgImportForm') as HTMLFormElement | null;
-
+  if (tcgImportForm) {
-      try {
+    tcgImportForm.addEventListener('submit', (e) => {
-        const resp = await fetch('/api/reindex', {
+      e.preventDefault();
-          method: 'POST',
+      const setName = (document.getElementById('tcgImportSetName') as HTMLInputElement).value.trim();
-          headers: { 'Content-Type': 'application/json' },
+      streamToProgress('/api/preload-tcgplayer', { setName }, `TCG Player Import: ${setName || '(none)'}`,
-          body: JSON.stringify(body),
+        document.getElementById('tcgImportRun') as HTMLButtonElement);
-        });
-
-        if (!resp.ok || !resp.body) {
-          progress.append(`Request failed: ${resp.status} ${resp.statusText}`);
-          progress.done('Failed');
-          return;
-        }
-
-        const reader = resp.body.getReader();
-        const decoder = new TextDecoder();
-        let buf = '';
-        while (true) {
-          const { value, done } = await reader.read();
-          if (done) break;
-          buf += decoder.decode(value, { stream: true });
-          const lines = buf.split('\n');
-          buf = lines.pop() ?? '';
-          for (const line of lines) progress.append(line);
-        }
-        if (buf) progress.append(buf);
-        progress.done('Done');
-      } catch (err: any) {
-        progress.append(`Error: ${err?.message || String(err)}`);
-        progress.done('Failed');
-      } finally {
-        runBtn.disabled = false;
-      }
     });
   }
 </script>

src/pages/api/preload-tcgplayer.ts

@@ -0,0 +1,43 @@
+import type { APIRoute } from 'astro';
+import { runImport } from '../../../scripts/preload-tcgplayer';
+
+export const POST: APIRoute = async ({ request }) => {
+    const { setName } = await request.json().catch(() => ({} as any));
+    const trimmed = typeof setName === 'string' ? setName.trim() : '';
+
+    const encoder = new TextEncoder();
+    const stream = new ReadableStream({
+        async start(controller) {
+            const log = (msg: string) => {
+                controller.enqueue(encoder.encode(msg + '\n'));
+            };
+
+            try {
+                if (!trimmed) {
+                    log('Set name is required.');
+                    return;
+                }
+
+                log(`Starting TCGPlayer import for set: "${trimmed}"`);
+                await runImport({ sets: [trimmed], log });
+                log('TCGPlayer import complete.');
+            } catch (e: any) {
+                const cause = e?.cause;
+                const causeMsg = cause?.message || (cause ? String(cause) : '');
+                log(`Error: ${e?.message || String(e)}`);
+                if (causeMsg) log(`Caused by: ${causeMsg}`);
+                console.error('TCGPlayer import error:', e);
+            } finally {
+                controller.close();
+            }
+        },
+    });
+
+    return new Response(stream, {
+        headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+            'Cache-Control': 'no-cache, no-transform',
+            'X-Accel-Buffering': 'no',
+        },
+    });
+};