import 'dotenv/config'; import * as schema from '../src/db/schema.ts'; import { db, ClosePool } from '../src/db/index.ts'; import fs from "node:fs/promises"; import path from "node:path"; import chalk from 'chalk'; import * as helper from './pokemon-helper.ts'; //import util from 'util'; async function syncTcgplayer(cardSets:string[] = []) { const productLines = [ "pokemon", "pokemon-japan" ]; // work from the available sets within the product line for (const productLine of productLines) { const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}}; const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { method: 'POST', headers: {'Content-Type': 'application/json',}, body: JSON.stringify(d), }); if (!response.ok) { console.error('Error notifying sync completion:', response.statusText); process.exit(1); } const data = await response.json(); const setNames = data.results[0].aggregations.setName; for (const setName of setNames) { let processSet = true; if (cardSets.length > 0) { processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase())); } if (processSet) { console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`)); await syncProductLine(productLine, "setName", setName.urlValue); } } } console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!')); } async function syncProductLine(productLine: string, field: string, fieldValue: string) { let start = 0; let size = 50; let total = 1000000; while (start < total) { console.log(` Fetching items ${start} to ${start + size} of ${total}...`); const d = { "algorithm":"sales_dismax", "from":start, "size":size, "filters":{ "term":{"productLineName":[productLine], [field]:[fieldValue]} , "range":{}, "match":{} }, "listingSearch":{ "context":{"cart":{}}, "filters":{"term":{ "sellerStatus":"Live", "channelId":0 }, "range":{ "quantity":{"gte":1} }, "exclude":{"channelExclusion":0} } }, "context":{ "cart":{}, "shippingCountry":"US", "userProfile":{} }, "settings":{ "useFuzzySearch":false, "didYouMean":{} }, "sort":{} }; //console.log(util.inspect(d, { depth: null })); //process.exit(1); const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(d), }); if (!response.ok) { console.error('Error notifying sync completion:', response.statusText); process.exit(1); } const data = await response.json(); total = data.results[0].totalResults; for (const item of data.results[0].results) { // Check if productId already exists and skip if it does (to avoid hitting the API too much) if (allProductIds.size > 0 && allProductIds.has(item.productId)) { continue; } console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); // Get product detail const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { method: 'GET', }); if (!detailResponse.ok) { console.error('Error fetching product details:', detailResponse.statusText); process.exit(1); } const detailData = await detailResponse.json(); await db.insert(schema.tcgcards).values({ productId: item.productId, productName: detailData.productName, //productName: cleanProductName(item.productName), rarityName: item.rarityName, productLineName: detailData.productLineName, productLineUrlName: detailData.productLineUrlName, productStatusId: detailData.productStatusId, productTypeId: detailData.productTypeId, productUrlName: detailData.productUrlName, setId: detailData.setId, shippingCategoryId: detailData.shippingCategoryId, sealed: detailData.sealed, sellerListable: detailData.sellerListable, foilOnly: detailData.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, energyType: detailData.customAttributes.energyType?.[0] || null, flavorText: detailData.customAttributes.flavorText || null, hp: helper.GetNumberOrNull(item.customAttributes.hp), number: detailData.customAttributes.number || '', releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, lowestPrice: detailData.lowestPrice, lowestPriceWithShipping: detailData.lowestPriceWithShipping, marketPrice: detailData.marketPrice, maxFulfillableQuantity: detailData.maxFulfillableQuantity, medianPrice: detailData.medianPrice, totalListings: item.totalListings, artist: detailData.formattedAttributes.Artist || null, }).onConflictDoUpdate({ target: schema.tcgcards.productId, set: { productName: detailData.productName, //productName: cleanProductName(item.productName), rarityName: item.rarityName, productLineName: detailData.productLineName, productLineUrlName: detailData.productLineUrlName, productStatusId: detailData.productStatusId, productTypeId: detailData.productTypeId, productUrlName: detailData.productUrlName, setId: detailData.setId, shippingCategoryId: detailData.shippingCategoryId, sealed: detailData.sealed, sellerListable: detailData.sellerListable, foilOnly: detailData.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, energyType: detailData.customAttributes.energyType?.[0] || null, flavorText: detailData.customAttributes.flavorText || null, hp: helper.GetNumberOrNull(item.customAttributes.hp), number: detailData.customAttributes.number || '', releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, lowestPrice: detailData.lowestPrice, lowestPriceWithShipping: detailData.lowestPriceWithShipping, marketPrice: detailData.marketPrice, maxFulfillableQuantity: detailData.maxFulfillableQuantity, medianPrice: detailData.medianPrice, totalListings: item.totalListings, artist: detailData.formattedAttributes.Artist || null, }, }); // console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`); // console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`); // console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`); // set is... await db.insert(schema.sets).values({ setId: detailData.setId, setCode: detailData.setCode, setName: detailData.setName, setUrlName: detailData.setUrlName, }).onConflictDoUpdate({ target: schema.sets.setId, set: { setCode: detailData.setCode, setName: detailData.setName, setUrlName: detailData.setUrlName, }, }); // skus are... for (const skuItem of detailData.skus) { await db.insert(schema.skus).values({ skuId: skuItem.sku, productId: detailData.productId, condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, }).onConflictDoUpdate({ target: schema.skus.skuId, set: { condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, }, }); } // get image if it doesn't already exist const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`); if (!await helper.FileExists(imagePath)) { const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`); if (imageResponse.ok) { const buffer = await imageResponse.arrayBuffer(); await fs.writeFile(imagePath, Buffer.from(buffer)); } else { console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText)); await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8'); } } // be nice to the API and not send too many requests in a short time await helper.Sleep(300); } start += size; } } // clear the log file await fs.rm('missing_images.log', { force: true }); let allProductIds = new Set(); const args = process.argv.slice(2); if (args.length === 0) { allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId))); await syncTcgplayer(); } else { await syncTcgplayer(args); } // update the card table with new/updated variants await helper.UpdateVariants(db); // index the card updates await helper.upsertCardCollection(db); await ClosePool();