tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[feat] tcg player import added to admin page

Browse Source
This commit is contained in:
Thad Miller
2026-05-28 16:12:48 -04:00
parent b0dbe7ced5
commit 2cf47d2b15
5 changed files with 246 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/pokemon-helper.ts
View File

@@ -208,7 +208,7 @@ export const upsertInventoryCollection = async (db:DBInstance, log: Logger = def
export const UpdateVariants = async (db:DBInstance) => {
export const UpdateVariants = async (db:DBInstance, log: Logger = (m) => console.log(m)) => {
const updates = await db.execute(sql`update cards as c
set
product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name,
@@ -232,7 +232,7 @@ where c.product_id = a.product_id and c.variant = a.variant and
c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
)
`);
console.log(`Updated ${updates.rowCount} rows in cards table`);
log(`Updated ${updates.rowCount} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
select t.product_id, b.variant,
@@ -245,9 +245,9 @@ join (select distinct product_id, variant from skus) b on t.product_id = b.produ
left join tcg_overrides o on t.product_id = o.product_id
where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
`);
console.log(`Inserted ${inserts.rowCount} rows into cards table`);
log(`Inserted ${inserts.rowCount} rows into cards table`);
const skuUpdates = await db.execute(sql`update skus s set card_id = c.card_id from cards c where s.product_id = c.product_id and s.variant = c.variant and s.card_id is distinct from c.card_id`);
console.log(`Updated ${skuUpdates.rowCount} rows in skus table`);
log(`Updated ${skuUpdates.rowCount} rows in skus table`);
}

213
scripts/preload-tcgplayer.ts
View File

@@ -1,58 +1,37 @@
import 'dotenv/config';
import * as schema from '../src/db/schema.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { db, ClosePool, type DBInstance } from '../src/db/index.ts';
import fs from "node:fs/promises";
import path from "node:path";
import chalk from 'chalk';
import { pathToFileURL } from 'node:url';
import * as helper from './pokemon-helper.ts';
//import util from 'util';
export type Logger = (msg: string) => void;
const consoleLogger: Logger = (m) => console.log(m);
export type RunImportOptions = {
sets?: string[];
log?: Logger;
runUpdateVariants?: boolean;
runCardUpsert?: boolean;
};
async function syncTcgplayer(cardSets:string[] = []) {
const productLines = [ "pokemon", "pokemon-japan" ];
// work from the available sets within the product line
for (const productLine of productLines) {
const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
method: 'POST',
headers: {'Content-Type': 'application/json',},
body: JSON.stringify(d),
});
if (!response.ok) {
console.error('Error notifying sync completion:', response.statusText);
process.exit(1);
}
const data = await response.json();
const setNames = data.results[0].aggregations.setName;
for (const setName of setNames) {
let processSet = true;
if (cardSets.length > 0) {
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
}
if (processSet) {
console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
await syncProductLine(productLine, "setName", setName.urlValue);
}
}
}
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
}
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
const syncProductLine = async (
database: DBInstance,
productLine: string,
field: string,
fieldValue: string,
allProductIds: Set<number>,
log: Logger,
) => {
let start = 0;
let size = 50;
const size = 50;
let total = 1000000;
while (start < total) {
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
log(` Fetching items ${start} to ${start + size} of ${total}...`);
const d = {
"algorithm": "sales_dismax",
@@ -61,46 +40,29 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
"filters": {
"term": { "productLineName": [productLine], [field]: [fieldValue] },
"range": {},
"match":{}
"match": {},
},
"listingSearch": {
"context": { "cart": {} },
"filters":{"term":{
"sellerStatus":"Live",
"channelId":0
"filters": {
"term": { "sellerStatus": "Live", "channelId": 0 },
"range": { "quantity": { "gte": 1 } },
"exclude": { "channelExclusion": 0 },
},
"range":{
"quantity":{"gte":1}
},
"exclude":{"channelExclusion":0}
}
},
"context":{
"cart":{},
"shippingCountry":"US",
"userProfile":{}
},
"settings":{
"useFuzzySearch":false,
"didYouMean":{}
},
"sort":{}
"context": { "cart": {}, "shippingCountry": "US", "userProfile": {} },
"settings": { "useFuzzySearch": false, "didYouMean": {} },
"sort": {},
};
//console.log(util.inspect(d, { depth: null }));
//process.exit(1);
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(d),
});
if (!response.ok) {
console.error('Error notifying sync completion:', response.statusText);
process.exit(1);
throw new Error(`TCGPlayer search request failed: ${response.status} ${response.statusText}`);
}
const data = await response.json();
@@ -108,28 +70,24 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
for (const item of data.results[0].results) {
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
continue;
}
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
log(` - ${item.productName} (ID: ${item.productId})`);
// Get product detail
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
method: 'GET',
});
if (!detailResponse.ok) {
console.error('Error fetching product details:', detailResponse.statusText);
process.exit(1);
throw new Error(`Error fetching product details for ${item.productId}: ${detailResponse.statusText}`);
}
const detailData = await detailResponse.json();
await db.insert(schema.tcgcards).values({
await database.insert(schema.tcgcards).values({
productId: item.productId,
productName: detailData.productName,
//productName: cleanProductName(item.productName),
rarityName: item.rarityName,
productLineName: detailData.productLineName,
productLineUrlName: detailData.productLineUrlName,
@@ -167,7 +125,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
target: schema.tcgcards.productId,
set: {
productName: detailData.productName,
//productName: cleanProductName(item.productName),
rarityName: item.rarityName,
productLineName: detailData.productLineName,
productLineUrlName: detailData.productLineUrlName,
@@ -204,11 +161,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
},
});
// console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
// console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
// console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
// set is...
await db.insert(schema.sets).values({
await database.insert(schema.sets).values({
setId: detailData.setId,
setCode: detailData.setCode,
setName: detailData.setName,
@@ -222,10 +175,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
},
});
// skus are...
for (const skuItem of detailData.skus) {
await db.insert(schema.skus).values({
await database.insert(schema.skus).values({
skuId: skuItem.sku,
productId: detailData.productId,
condition: skuItem.condition,
@@ -241,7 +192,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
});
}
// get image if it doesn't already exist
const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
if (!await helper.FileExists(imagePath)) {
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
@@ -249,37 +199,88 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
const buffer = await imageResponse.arrayBuffer();
await fs.writeFile(imagePath, Buffer.from(buffer));
} else {
console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`);
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
}
}
// be nice to the API and not send too many requests in a short time
await helper.Sleep(300);
}
start += size;
}
};
const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set<number>, log: Logger) => {
const productLines = ["pokemon", "pokemon-japan"];
for (const productLine of productLines) {
const d = {
"algorithm": "sales_dismax", "from": 0, "size": 1,
"filters": { "term": { "productLineName": [productLine] } },
"settings": { "useFuzzySearch": false },
};
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(d),
});
if (!response.ok) {
throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`);
}
const data = await response.json();
const setNames = data.results[0].aggregations.setName;
for (const setName of setNames) {
let processSet = true;
if (cardSets.length > 0) {
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
}
if (processSet) {
log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`);
await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log);
}
}
}
// clear the log file
log('All TCGPlayer data synchronized successfully!');
};
export const runImport = async (opts: RunImportOptions = {}) => {
const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts;
await fs.rm('missing_images.log', { force: true });
let allProductIds = new Set();
// When no set filter is provided, skip productIds already in the cards table
// (matches the CLI script's "no args" behavior).
const allProductIds = sets.length === 0
? new Set<number>(
await db.select({ productId: schema.cards.productId }).from(schema.cards)
.then(rows => rows.map(row => row.productId))
)
: new Set<number>();
await syncTcgplayer(db, sets, allProductIds, log);
if (runUpdateVariants) {
log('Updating card variants...');
await helper.UpdateVariants(db, log);
}
if (runCardUpsert) {
log('Reindexing "cards" collection...');
await helper.upsertCardCollection(db, log);
}
};
// CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage.
const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
if (isCli) {
const args = process.argv.slice(2);
if (args.length === 0) {
allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
await syncTcgplayer();
}
else {
await syncTcgplayer(args);
}
// update the card table with new/updated variants
await helper.UpdateVariants(db);
// index the card updates
await helper.upsertCardCollection(db);
await runImport({ sets: args });
await ClosePool();
}

2
src/middleware.ts
View File

@@ -10,7 +10,7 @@ declare global {
}
const isProtectedRoute = createRouteMatcher(['/pokemon']);
const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex']);
const isAdminRoute = createRouteMatcher(['/admin', '/api/reindex', '/api/preload-tcgplayer']);
const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK";
const ADMIN_ORG_IDS = new Set([

71
src/pages/admin.astro
View File

@@ -53,6 +53,29 @@ import Footer from '../components/Footer.astro';
</div>
</div>
</div>
<div class="accordion-item">
<h2 class="accordion-header" id="tcgImportHeading">
<button class="accordion-button collapsed" type="button" data-bs-toggle="collapse"
data-bs-target="#tcgImportCollapse" aria-expanded="false" aria-controls="tcgImportCollapse">
TCG Player Import
</button>
</h2>
<div id="tcgImportCollapse" class="accordion-collapse collapse" aria-labelledby="tcgImportHeading"
data-bs-parent="#adminAccordion">
<div class="accordion-body">
<form id="tcgImportForm">
<div class="mb-3">
<label for="tcgImportSetName" class="form-label">Set Name</label>
<input type="text" class="form-control" id="tcgImportSetName" name="setName"
placeholder="e.g. Surging Sparks" autocomplete="off" required />
<div class="form-text">Matches any set whose name contains this text (case-insensitive).</div>
</div>
<button type="submit" class="btn btn-purple" id="tcgImportRun">Run Import</button>
</form>
</div>
</div>
</div>
</div>
</div>
</div>
@@ -137,35 +160,21 @@ import Footer from '../components/Footer.astro';
window.AdminProgress = { open: openProgress };
// Reindex form wiring
const form = document.getElementById('reindexForm') as HTMLFormElement | null;
if (form) {
form.addEventListener('submit', async (e) => {
e.preventDefault();
const runBtn = document.getElementById('reindexRun') as HTMLButtonElement;
const body = {
cards: (document.getElementById('reindexCards') as HTMLInputElement).checked,
skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked,
inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked,
recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked,
};
// Stream a POST JSON request line-by-line into a progress modal.
const streamToProgress = async (url: string, body: unknown, title: string, runBtn: HTMLButtonElement) => {
runBtn.disabled = true;
const progress = await window.AdminProgress.open('Reindex');
const progress = await window.AdminProgress.open(title);
try {
const resp = await fetch('/api/reindex', {
const resp = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (!resp.ok || !resp.body) {
progress.append(`Request failed: ${resp.status} ${resp.statusText}`);
progress.done('Failed');
return;
}
const reader = resp.body.getReader();
const decoder = new TextDecoder();
let buf = '';
@@ -185,6 +194,32 @@ import Footer from '../components/Footer.astro';
} finally {
runBtn.disabled = false;
}
};
// Reindex form wiring
const reindexForm = document.getElementById('reindexForm') as HTMLFormElement | null;
if (reindexForm) {
reindexForm.addEventListener('submit', (e) => {
e.preventDefault();
const body = {
cards: (document.getElementById('reindexCards') as HTMLInputElement).checked,
skus: (document.getElementById('reindexSkus') as HTMLInputElement).checked,
inventory: (document.getElementById('reindexInventory') as HTMLInputElement).checked,
recreate: (document.getElementById('reindexRecreate') as HTMLInputElement).checked,
};
streamToProgress('/api/reindex', body, 'Reindex',
document.getElementById('reindexRun') as HTMLButtonElement);
});
}
// TCG Player import form wiring
const tcgImportForm = document.getElementById('tcgImportForm') as HTMLFormElement | null;
if (tcgImportForm) {
tcgImportForm.addEventListener('submit', (e) => {
e.preventDefault();
const setName = (document.getElementById('tcgImportSetName') as HTMLInputElement).value.trim();
streamToProgress('/api/preload-tcgplayer', { setName }, `TCG Player Import: ${setName || '(none)'}`,
document.getElementById('tcgImportRun') as HTMLButtonElement);
});
}
</script>

43
src/pages/api/preload-tcgplayer.ts Normal file
View File

@@ -0,0 +1,43 @@
import type { APIRoute } from 'astro';
import { runImport } from '../../../scripts/preload-tcgplayer';
export const POST: APIRoute = async ({ request }) => {
const { setName } = await request.json().catch(() => ({} as any));
const trimmed = typeof setName === 'string' ? setName.trim() : '';
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
const log = (msg: string) => {
controller.enqueue(encoder.encode(msg + '\n'));
};
try {
if (!trimmed) {
log('Set name is required.');
return;
}
log(`Starting TCGPlayer import for set: "${trimmed}"`);
await runImport({ sets: [trimmed], log });
log('TCGPlayer import complete.');
} catch (e: any) {
const cause = e?.cause;
const causeMsg = cause?.message || (cause ? String(cause) : '');
log(`Error: ${e?.message || String(e)}`);
if (causeMsg) log(`Caused by: ${causeMsg}`);
console.error('TCGPlayer import error:', e);
} finally {
controller.close();
}
},
});
return new Response(stream, {
headers: {
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'no-cache, no-transform',
'X-Accel-Buffering': 'no',
},
});
};