import 'dotenv/config'; import * as schema from '../src/db/schema.ts'; import { db, ClosePool, type DBInstance } from '../src/db/index.ts'; import fs from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from 'node:url'; import * as helper from './pokemon-helper.ts'; export type Logger = (msg: string) => void; const consoleLogger: Logger = (m) => console.log(m); export type RunImportOptions = { sets?: string[]; log?: Logger; runUpdateVariants?: boolean; runCardUpsert?: boolean; }; const syncProductLine = async ( database: DBInstance, productLine: string, field: string, fieldValue: string, allProductIds: Set, log: Logger, ) => { let start = 0; const size = 50; let total = 1000000; while (start < total) { log(` Fetching items ${start} to ${start + size} of ${total}...`); const d = { "algorithm": "sales_dismax", "from": start, "size": size, "filters": { "term": { "productLineName": [productLine], [field]: [fieldValue] }, "range": {}, "match": {}, }, "listingSearch": { "context": { "cart": {} }, "filters": { "term": { "sellerStatus": "Live", "channelId": 0 }, "range": { "quantity": { "gte": 1 } }, "exclude": { "channelExclusion": 0 }, }, }, "context": { "cart": {}, "shippingCountry": "US", "userProfile": {} }, "settings": { "useFuzzySearch": false, "didYouMean": {} }, "sort": {}, }; const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(d), }); if (!response.ok) { throw new Error(`TCGPlayer search request failed: ${response.status} ${response.statusText}`); } const data = await response.json(); total = data.results[0].totalResults; for (const item of data.results[0].results) { if (allProductIds.size > 0 && allProductIds.has(item.productId)) { continue; } log(` - ${item.productName} (ID: ${item.productId})`); const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { method: 'GET', }); if (!detailResponse.ok) { throw new Error(`Error fetching product details for ${item.productId}: ${detailResponse.statusText}`); } const detailData = await detailResponse.json(); await database.insert(schema.tcgcards).values({ productId: item.productId, productName: detailData.productName, rarityName: item.rarityName, productLineName: detailData.productLineName, productLineUrlName: detailData.productLineUrlName, productStatusId: detailData.productStatusId, productTypeId: detailData.productTypeId, productUrlName: detailData.productUrlName, setId: detailData.setId, shippingCategoryId: detailData.shippingCategoryId, sealed: detailData.sealed, sellerListable: detailData.sellerListable, foilOnly: detailData.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, energyType: detailData.customAttributes.energyType?.[0] || null, flavorText: detailData.customAttributes.flavorText || null, hp: helper.GetNumberOrNull(item.customAttributes.hp), number: detailData.customAttributes.number || '', releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, lowestPrice: detailData.lowestPrice, lowestPriceWithShipping: detailData.lowestPriceWithShipping, marketPrice: detailData.marketPrice, maxFulfillableQuantity: detailData.maxFulfillableQuantity, medianPrice: detailData.medianPrice, totalListings: item.totalListings, artist: detailData.formattedAttributes.Artist || null, }).onConflictDoUpdate({ target: schema.tcgcards.productId, set: { productName: detailData.productName, rarityName: item.rarityName, productLineName: detailData.productLineName, productLineUrlName: detailData.productLineUrlName, productStatusId: detailData.productStatusId, productTypeId: detailData.productTypeId, productUrlName: detailData.productUrlName, setId: detailData.setId, shippingCategoryId: detailData.shippingCategoryId, sealed: detailData.sealed, sellerListable: detailData.sellerListable, foilOnly: detailData.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, energyType: detailData.customAttributes.energyType?.[0] || null, flavorText: detailData.customAttributes.flavorText || null, hp: helper.GetNumberOrNull(item.customAttributes.hp), number: detailData.customAttributes.number || '', releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, lowestPrice: detailData.lowestPrice, lowestPriceWithShipping: detailData.lowestPriceWithShipping, marketPrice: detailData.marketPrice, maxFulfillableQuantity: detailData.maxFulfillableQuantity, medianPrice: detailData.medianPrice, totalListings: item.totalListings, artist: detailData.formattedAttributes.Artist || null, }, }); await database.insert(schema.sets).values({ setId: detailData.setId, setCode: detailData.setCode, setName: detailData.setName, setUrlName: detailData.setUrlName, }).onConflictDoUpdate({ target: schema.sets.setId, set: { setCode: detailData.setCode, setName: detailData.setName, setUrlName: detailData.setUrlName, }, }); for (const skuItem of detailData.skus) { await database.insert(schema.skus).values({ skuId: skuItem.sku, productId: detailData.productId, condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, }).onConflictDoUpdate({ target: schema.skus.skuId, set: { condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, }, }); } const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`); if (!await helper.FileExists(imagePath)) { const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`); if (imageResponse.ok) { const buffer = await imageResponse.arrayBuffer(); await fs.writeFile(imagePath, Buffer.from(buffer)); } else { log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`); await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8'); } } await helper.Sleep(300); } start += size; } }; const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set, log: Logger) => { const productLines = ["pokemon", "pokemon-japan"]; for (const productLine of productLines) { const d = { "algorithm": "sales_dismax", "from": 0, "size": 1, "filters": { "term": { "productLineName": [productLine] } }, "settings": { "useFuzzySearch": false }, }; const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(d), }); if (!response.ok) { throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`); } const data = await response.json(); const setNames = data.results[0].aggregations.setName; for (const setName of setNames) { let processSet = true; if (cardSets.length > 0) { processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase())); } if (processSet) { log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`); await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log); } } } log('All TCGPlayer data synchronized successfully!'); }; export const runImport = async (opts: RunImportOptions = {}) => { const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts; await fs.rm('missing_images.log', { force: true }); // When no set filter is provided, skip productIds already in the cards table // (matches the CLI script's "no args" behavior). const allProductIds = sets.length === 0 ? new Set( await db.select({ productId: schema.cards.productId }).from(schema.cards) .then(rows => rows.map(row => row.productId)) ) : new Set(); await syncTcgplayer(db, sets, allProductIds, log); if (runUpdateVariants) { log('Updating card variants...'); await helper.UpdateVariants(db, log); } if (runCardUpsert) { log('Reindexing "cards" collection...'); await helper.upsertCardCollection(db, log); } }; // CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage. const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; if (isCli) { const args = process.argv.slice(2); await runImport({ sets: args }); await ClosePool(); }