From 2cf47d2b157b93c8ecefcd60600d227516cb074f Mon Sep 17 00:00:00 2001 From: Thad Miller Date: Thu, 28 May 2026 16:12:48 -0400 Subject: [PATCH] [feat] tcg player import added to admin page --- scripts/pokemon-helper.ts | 8 +- scripts/preload-tcgplayer.ts | 247 +++++++++++++++-------------- src/middleware.ts | 2 +- src/pages/admin.astro | 113 ++++++++----- src/pages/api/preload-tcgplayer.ts | 43 +++++ 5 files changed, 246 insertions(+), 167 deletions(-) create mode 100644 src/pages/api/preload-tcgplayer.ts diff --git a/scripts/pokemon-helper.ts b/scripts/pokemon-helper.ts index 2d7f021..82558c1 100644 --- a/scripts/pokemon-helper.ts +++ b/scripts/pokemon-helper.ts @@ -208,7 +208,7 @@ export const upsertInventoryCollection = async (db:DBInstance, log: Logger = def -export const UpdateVariants = async (db:DBInstance) => { +export const UpdateVariants = async (db:DBInstance, log: Logger = (m) => console.log(m)) => { const updates = await db.execute(sql`update cards as c set product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, @@ -232,7 +232,7 @@ where c.product_id = a.product_id and c.variant = a.variant and c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist ) `); - console.log(`Updated ${updates.rowCount} rows in cards table`); + log(`Updated ${updates.rowCount} rows in cards table`); const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist) select t.product_id, b.variant, @@ -245,9 +245,9 @@ join (select distinct product_id, variant from skus) b on t.product_id = b.produ left join tcg_overrides o on t.product_id = o.product_id where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant) `); - console.log(`Inserted ${inserts.rowCount} rows into cards table`); + log(`Inserted ${inserts.rowCount} rows into cards table`); const skuUpdates = await db.execute(sql`update skus s set card_id = c.card_id from cards c where s.product_id = c.product_id and s.variant = c.variant and s.card_id is distinct from c.card_id`); - console.log(`Updated ${skuUpdates.rowCount} rows in skus table`); + log(`Updated ${skuUpdates.rowCount} rows in skus table`); } diff --git a/scripts/preload-tcgplayer.ts b/scripts/preload-tcgplayer.ts index 6a5fec4..1ad5b7a 100644 --- a/scripts/preload-tcgplayer.ts +++ b/scripts/preload-tcgplayer.ts @@ -1,106 +1,68 @@ import 'dotenv/config'; import * as schema from '../src/db/schema.ts'; -import { db, ClosePool } from '../src/db/index.ts'; +import { db, ClosePool, type DBInstance } from '../src/db/index.ts'; import fs from "node:fs/promises"; import path from "node:path"; -import chalk from 'chalk'; +import { pathToFileURL } from 'node:url'; import * as helper from './pokemon-helper.ts'; -//import util from 'util'; + +export type Logger = (msg: string) => void; +const consoleLogger: Logger = (m) => console.log(m); + +export type RunImportOptions = { + sets?: string[]; + log?: Logger; + runUpdateVariants?: boolean; + runCardUpsert?: boolean; +}; -async function syncTcgplayer(cardSets:string[] = []) { - - const productLines = [ "pokemon", "pokemon-japan" ]; - - // work from the available sets within the product line - for (const productLine of productLines) { - const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}}; - - const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { - method: 'POST', - headers: {'Content-Type': 'application/json',}, - body: JSON.stringify(d), - }); - if (!response.ok) { - console.error('Error notifying sync completion:', response.statusText); - process.exit(1); - } - const data = await response.json(); - - const setNames = data.results[0].aggregations.setName; - for (const setName of setNames) { - let processSet = true; - if (cardSets.length > 0) { - processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase())); - } - if (processSet) { - console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`)); - await syncProductLine(productLine, "setName", setName.urlValue); - } - } - - } - - console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!')); -} - - -async function syncProductLine(productLine: string, field: string, fieldValue: string) { +const syncProductLine = async ( + database: DBInstance, + productLine: string, + field: string, + fieldValue: string, + allProductIds: Set, + log: Logger, +) => { let start = 0; - let size = 50; + const size = 50; let total = 1000000; while (start < total) { - console.log(` Fetching items ${start} to ${start + size} of ${total}...`); + log(` Fetching items ${start} to ${start + size} of ${total}...`); const d = { - "algorithm":"sales_dismax", - "from":start, - "size":size, - "filters":{ - "term":{"productLineName":[productLine], [field]:[fieldValue]} , - "range":{}, - "match":{} - }, - "listingSearch":{ - "context":{"cart":{}}, - "filters":{"term":{ - "sellerStatus":"Live", - "channelId":0 - }, - "range":{ - "quantity":{"gte":1} - }, - "exclude":{"channelExclusion":0} - } - }, - "context":{ - "cart":{}, - "shippingCountry":"US", - "userProfile":{} - }, - "settings":{ - "useFuzzySearch":false, - "didYouMean":{} - }, - "sort":{} - }; - - //console.log(util.inspect(d, { depth: null })); - //process.exit(1); + "algorithm": "sales_dismax", + "from": start, + "size": size, + "filters": { + "term": { "productLineName": [productLine], [field]: [fieldValue] }, + "range": {}, + "match": {}, + }, + "listingSearch": { + "context": { "cart": {} }, + "filters": { + "term": { "sellerStatus": "Live", "channelId": 0 }, + "range": { "quantity": { "gte": 1 } }, + "exclude": { "channelExclusion": 0 }, + }, + }, + "context": { "cart": {}, "shippingCountry": "US", "userProfile": {} }, + "settings": { "useFuzzySearch": false, "didYouMean": {} }, + "sort": {}, + }; const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, + headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(d), }); if (!response.ok) { - console.error('Error notifying sync completion:', response.statusText); - process.exit(1); + throw new Error(`TCGPlayer search request failed: ${response.status} ${response.statusText}`); } const data = await response.json(); @@ -108,28 +70,24 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s for (const item of data.results[0].results) { - // Check if productId already exists and skip if it does (to avoid hitting the API too much) if (allProductIds.size > 0 && allProductIds.has(item.productId)) { continue; } - console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); + log(` - ${item.productName} (ID: ${item.productId})`); - // Get product detail const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { method: 'GET', }); if (!detailResponse.ok) { - console.error('Error fetching product details:', detailResponse.statusText); - process.exit(1); + throw new Error(`Error fetching product details for ${item.productId}: ${detailResponse.statusText}`); } const detailData = await detailResponse.json(); - await db.insert(schema.tcgcards).values({ + await database.insert(schema.tcgcards).values({ productId: item.productId, productName: detailData.productName, - //productName: cleanProductName(item.productName), rarityName: item.rarityName, productLineName: detailData.productLineName, productLineUrlName: detailData.productLineUrlName, @@ -167,7 +125,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s target: schema.tcgcards.productId, set: { productName: detailData.productName, - //productName: cleanProductName(item.productName), rarityName: item.rarityName, productLineName: detailData.productLineName, productLineUrlName: detailData.productLineUrlName, @@ -204,11 +161,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s }, }); - // console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`); - // console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`); - // console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`); - // set is... - await db.insert(schema.sets).values({ + await database.insert(schema.sets).values({ setId: detailData.setId, setCode: detailData.setCode, setName: detailData.setName, @@ -222,10 +175,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s }, }); - // skus are... for (const skuItem of detailData.skus) { - - await db.insert(schema.skus).values({ + await database.insert(schema.skus).values({ skuId: skuItem.sku, productId: detailData.productId, condition: skuItem.condition, @@ -241,7 +192,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s }); } - // get image if it doesn't already exist const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`); if (!await helper.FileExists(imagePath)) { const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`); @@ -249,37 +199,88 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s const buffer = await imageResponse.arrayBuffer(); await fs.writeFile(imagePath, Buffer.from(buffer)); } else { - console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText)); + log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`); await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8'); } } - // be nice to the API and not send too many requests in a short time await helper.Sleep(300); - } start += size; } +}; + + +const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set, log: Logger) => { + const productLines = ["pokemon", "pokemon-japan"]; + + for (const productLine of productLines) { + const d = { + "algorithm": "sales_dismax", "from": 0, "size": 1, + "filters": { "term": { "productLineName": [productLine] } }, + "settings": { "useFuzzySearch": false }, + }; + + const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(d), + }); + if (!response.ok) { + throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`); + } + const data = await response.json(); + + const setNames = data.results[0].aggregations.setName; + for (const setName of setNames) { + let processSet = true; + if (cardSets.length > 0) { + processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase())); + } + if (processSet) { + log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`); + await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log); + } + } + } + + log('All TCGPlayer data synchronized successfully!'); +}; + + +export const runImport = async (opts: RunImportOptions = {}) => { + const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts; + + await fs.rm('missing_images.log', { force: true }); + + // When no set filter is provided, skip productIds already in the cards table + // (matches the CLI script's "no args" behavior). + const allProductIds = sets.length === 0 + ? new Set( + await db.select({ productId: schema.cards.productId }).from(schema.cards) + .then(rows => rows.map(row => row.productId)) + ) + : new Set(); + + await syncTcgplayer(db, sets, allProductIds, log); + + if (runUpdateVariants) { + log('Updating card variants...'); + await helper.UpdateVariants(db, log); + } + + if (runCardUpsert) { + log('Reindexing "cards" collection...'); + await helper.upsertCardCollection(db, log); + } +}; + + +// CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage. +const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (isCli) { + const args = process.argv.slice(2); + await runImport({ sets: args }); + await ClosePool(); } - -// clear the log file -await fs.rm('missing_images.log', { force: true }); -let allProductIds = new Set(); - -const args = process.argv.slice(2); -if (args.length === 0) { - allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId))); - await syncTcgplayer(); -} -else { - await syncTcgplayer(args); -} - -// update the card table with new/updated variants -await helper.UpdateVariants(db); - -// index the card updates -await helper.upsertCardCollection(db); - -await ClosePool(); diff --git a/src/middleware.ts b/src/middleware.ts index a05806a..914744b 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -10,7 +10,7 @@ declare global { } const isProtectedRoute = createRouteMatcher(['/pokemon']); -const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex']); +const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex', '/api/preload-tcgplayer']); const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK"; const ADMIN_ORG_IDS = new Set([ diff --git a/src/pages/admin.astro b/src/pages/admin.astro index 0a5efab..328465c 100644 --- a/src/pages/admin.astro +++ b/src/pages/admin.astro @@ -53,6 +53,29 @@ import Footer from '../components/Footer.astro'; + +
+

+ +

+
+
+
+
+ + +
Matches any set whose name contains this text (case-insensitive).
+
+ +
+
+
+
@@ -137,54 +160,66 @@ import Footer from '../components/Footer.astro'; window.AdminProgress = { open: openProgress }; + // Stream a POST JSON request line-by-line into a progress modal. + const streamToProgress = async (url: string, body: unknown, title: string, runBtn: HTMLButtonElement) => { + runBtn.disabled = true; + const progress = await window.AdminProgress.open(title); + try { + const resp = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!resp.ok || !resp.body) { + progress.append(`Request failed: ${resp.status} ${resp.statusText}`); + progress.done('Failed'); + return; + } + const reader = resp.body.getReader(); + const decoder = new TextDecoder(); + let buf = ''; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + const lines = buf.split('\n'); + buf = lines.pop() ?? ''; + for (const line of lines) progress.append(line); + } + if (buf) progress.append(buf); + progress.done('Done'); + } catch (err: any) { + progress.append(`Error: ${err?.message || String(err)}`); + progress.done('Failed'); + } finally { + runBtn.disabled = false; + } + }; + // Reindex form wiring - const form = document.getElementById('reindexForm') as HTMLFormElement | null; - if (form) { - form.addEventListener('submit', async (e) => { + const reindexForm = document.getElementById('reindexForm') as HTMLFormElement | null; + if (reindexForm) { + reindexForm.addEventListener('submit', (e) => { e.preventDefault(); - const runBtn = document.getElementById('reindexRun') as HTMLButtonElement; const body = { cards: (document.getElementById('reindexCards') as HTMLInputElement).checked, skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked, inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked, recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked, }; + streamToProgress('/api/reindex', body, 'Reindex', + document.getElementById('reindexRun') as HTMLButtonElement); + }); + } - runBtn.disabled = true; - const progress = await window.AdminProgress.open('Reindex'); - - try { - const resp = await fetch('/api/reindex', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - - if (!resp.ok || !resp.body) { - progress.append(`Request failed: ${resp.status} ${resp.statusText}`); - progress.done('Failed'); - return; - } - - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - let buf = ''; - while (true) { - const { value, done } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - const lines = buf.split('\n'); - buf = lines.pop() ?? ''; - for (const line of lines) progress.append(line); - } - if (buf) progress.append(buf); - progress.done('Done'); - } catch (err: any) { - progress.append(`Error: ${err?.message || String(err)}`); - progress.done('Failed'); - } finally { - runBtn.disabled = false; - } + // TCG Player import form wiring + const tcgImportForm = document.getElementById('tcgImportForm') as HTMLFormElement | null; + if (tcgImportForm) { + tcgImportForm.addEventListener('submit', (e) => { + e.preventDefault(); + const setName = (document.getElementById('tcgImportSetName') as HTMLInputElement).value.trim(); + streamToProgress('/api/preload-tcgplayer', { setName }, `TCG Player Import: ${setName || '(none)'}`, + document.getElementById('tcgImportRun') as HTMLButtonElement); }); } diff --git a/src/pages/api/preload-tcgplayer.ts b/src/pages/api/preload-tcgplayer.ts new file mode 100644 index 0000000..281de28 --- /dev/null +++ b/src/pages/api/preload-tcgplayer.ts @@ -0,0 +1,43 @@ +import type { APIRoute } from 'astro'; +import { runImport } from '../../../scripts/preload-tcgplayer'; + +export const POST: APIRoute = async ({ request }) => { + const { setName } = await request.json().catch(() => ({} as any)); + const trimmed = typeof setName === 'string' ? setName.trim() : ''; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + const log = (msg: string) => { + controller.enqueue(encoder.encode(msg + '\n')); + }; + + try { + if (!trimmed) { + log('Set name is required.'); + return; + } + + log(`Starting TCGPlayer import for set: "${trimmed}"`); + await runImport({ sets: [trimmed], log }); + log('TCGPlayer import complete.'); + } catch (e: any) { + const cause = e?.cause; + const causeMsg = cause?.message || (cause ? String(cause) : ''); + log(`Error: ${e?.message || String(e)}`); + if (causeMsg) log(`Caused by: ${causeMsg}`); + console.error('TCGPlayer import error:', e); + } finally { + controller.close(); + } + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'no-cache, no-transform', + 'X-Accel-Buffering': 'no', + }, + }); +};