import 'dotenv/config'; import { drizzle } from 'drizzle-orm/mysql2'; import mysql from 'mysql2/promise'; import * as schema from '../src/db/schema.ts'; import fs from "node:fs/promises"; import path from "node:path"; import { eq } from 'drizzle-orm'; import chalk from 'chalk'; //import util from 'util'; async function syncTcgplayer() { // const productLines = [ // { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }, // { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] } // ]; const productLines = [ { name: "pokemon-japan", cardType: ["Dragon", "Colorless", "Energy"] } ]; for (const productLine of productLines) { for (const [key, values] of Object.entries(productLine)) { if (key === "name") continue; for (const value of values) { console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`); await syncProductLineEnergyType(productLine.name, key, value); } } } console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!')); } function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } async function fileExists(path: string): Promise { try { await fs.access(path); return true; } catch { return false; } } async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) { let start = 0; let size = 50; let total = 1000000; while (start < total) { console.log(` Fetching items ${start} to ${start + size} of ${total}...`); let d = { "algorithm":"sales_dismax", "from":start, "size":size, "filters":{ "term":{"productLineName":[productLine]}, "range":{}, "match":{} }, "listingSearch":{ "context":{"cart":{}}, "filters":{"term":{ "sellerStatus":"Live", "channelId":0 }, "range":{ "quantity":{"gte":1} }, "exclude":{"channelExclusion":0} } }, "context":{ "cart":{}, "shippingCountry":"US", "userProfile":{} }, "settings":{ "useFuzzySearch":false, "didYouMean":{} }, "sort":{} }; d.filters.term[field] = [fieldValue]; //console.log(util.inspect(d, { depth: null })); //process.exit(1); const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(d), }); if (!response.ok) { console.error('Error notifying sync completion:', response.statusText); process.exit(1); } const data = await response.json(); total = data.results[0].totalResults; //console.log(data); const poolConnection = mysql.createPool({ uri: process.env.DATABASE_URL, }); const db = drizzle(poolConnection, { schema, mode: 'default' }); for (const item of data.results[0].results) { console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); await db.insert(schema.cards).values({ productId: item.productId, productName: item.productName, rarityName: item.rarityName, productLineName: item.productLineName, productLineUrlName: item.productLineUrlName, productStatusId: item.productStatusId, productTypeId: item.productTypeId, productUrlName: item.productUrlName, setId: item.setId, shippingCategoryId: item.shippingCategoryId, sealed: item.sealed, sellerListable: item.sellerListable, foilOnly: item.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, energyType: item.customAttributes.energyType?.[0] || null, flavorText: item.customAttributes.flavorText || null, hp: item.customAttributes.hp || 0, number: item.customAttributes.number || '', releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, lowestPrice: item.lowestPrice, lowestPriceWithShipping: item.lowestPriceWithShipping, marketPrice: item.marketPrice, maxFulfillableQuantity: item.maxFulfillableQuantity, medianPrice: item.medianPrice, totalListings: item.totalListings, }).onDuplicateKeyUpdate({ set: { productName: item.productName, rarityName: item.rarityName, productLineName: item.productLineName, productLineUrlName: item.productLineUrlName, productStatusId: item.productStatusId, productTypeId: item.productTypeId, productUrlName: item.productUrlName, setId: item.setId, shippingCategoryId: item.shippingCategoryId, sealed: item.sealed, sellerListable: item.sellerListable, foilOnly: item.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, energyType: item.customAttributes.energyType?.[0] || null, flavorText: item.customAttributes.flavorText || null, hp: item.customAttributes.hp || 0, number: item.customAttributes.number || '', releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, lowestPrice: item.lowestPrice, lowestPriceWithShipping: item.lowestPriceWithShipping, marketPrice: item.marketPrice, maxFulfillableQuantity: item.maxFulfillableQuantity, medianPrice: item.medianPrice, totalListings: item.totalListings, }, }); // before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls. const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId)); const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000); if (hasRecentSku) { console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data')); await sleep(100); continue; } // Get product detail const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { method: 'GET', }); if (!detailResponse.ok) { console.error('Error fetching product details:', detailResponse.statusText); process.exit(1); } const detailData = await detailResponse.json(); await db.insert(schema.sets).values({ setId: detailData.setId, setCode: detailData.setCode, setName: detailData.setName, setUrlName: detailData.setUrlName, }).onDuplicateKeyUpdate({ set: { setCode: detailData.setCode, setName: detailData.setName, setUrlName: detailData.setUrlName, }, }); // skus are... const skuArray = detailData.skus.map((sku: any) => sku.sku); //console.log(detailData.skus); //console.log(skuArray); // get pricing for skus const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ skuIds: skuArray }), }); if (!skuResponse.ok) { console.error('Error fetching SKU pricing:', skuResponse.statusText); process.exit(1); } const skuData = await skuResponse.json(); let skuMap = new Map(); for (const skuItem of skuData) { skuMap.set(skuItem.skuId, skuItem); } for (const skuItem of detailData.skus) { const pricing = skuMap.get(skuItem.sku); //console.log(pricing); await db.insert(schema.skus).values({ skuId: skuItem.sku, productId: detailData.productId, condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null, highestPrice: pricing?.highestPrice || null, lowestPrice: pricing?.lowestPrice || null, marketPrice: pricing?.marketPrice || null, priceCount: pricing?.priceCount || 0, }).onDuplicateKeyUpdate({ set: { condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null, highestPrice: pricing?.highestPrice || null, lowestPrice: pricing?.lowestPrice || null, marketPrice: pricing?.marketPrice || null, priceCount: pricing?.priceCount || 0, }, }); } // get image if it doesn't already exist const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`); if (!await fileExists(imagePath)) { const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`); if (imageResponse.ok) { const buffer = await imageResponse.arrayBuffer(); await fs.writeFile(imagePath, Buffer.from(buffer)); } else { console.error('Error fetching product image:', imageResponse.statusText); } } // be nice to the API and not send too many requests in a short time await sleep(100); } await poolConnection.end(); start += size; } } syncTcgplayer();