309 lines
11 KiB
TypeScript
309 lines
11 KiB
TypeScript
import 'dotenv/config';
|
|
import { drizzle } from 'drizzle-orm/mysql2';
|
|
import mysql from 'mysql2/promise';
|
|
import * as schema from '../src/db/schema.ts';
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { eq } from 'drizzle-orm';
|
|
import chalk from 'chalk';
|
|
//import util from 'util';
|
|
|
|
|
|
async function syncTcgplayer() {
|
|
|
|
// const productLines = [
|
|
// { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
|
|
// { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }
|
|
// ];
|
|
|
|
const productLines = [
|
|
{ name: "pokemon-japan", cardType: ["Dragon", "Colorless", "Energy"] }
|
|
];
|
|
|
|
for (const productLine of productLines) {
|
|
for (const [key, values] of Object.entries(productLine)) {
|
|
if (key === "name") continue;
|
|
for (const value of values) {
|
|
console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
|
|
await syncProductLineEnergyType(productLine.name, key, value);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
|
|
}
|
|
|
|
function sleep(ms: number) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function fileExists(path: string): Promise<boolean> {
|
|
try {
|
|
await fs.access(path);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) {
|
|
let start = 0;
|
|
let size = 50;
|
|
let total = 1000000;
|
|
|
|
while (start < total) {
|
|
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
|
|
|
|
|
let d = {
|
|
"algorithm":"sales_dismax",
|
|
"from":start,
|
|
"size":size,
|
|
"filters":{
|
|
"term":{"productLineName":[productLine]},
|
|
"range":{},
|
|
"match":{}
|
|
},
|
|
"listingSearch":{
|
|
"context":{"cart":{}},
|
|
"filters":{"term":{
|
|
"sellerStatus":"Live",
|
|
"channelId":0
|
|
},
|
|
"range":{
|
|
"quantity":{"gte":1}
|
|
},
|
|
"exclude":{"channelExclusion":0}
|
|
}
|
|
},
|
|
"context":{
|
|
"cart":{},
|
|
"shippingCountry":"US",
|
|
"userProfile":{}
|
|
},
|
|
"settings":{
|
|
"useFuzzySearch":false,
|
|
"didYouMean":{}
|
|
},
|
|
"sort":{}
|
|
};
|
|
d.filters.term[field] = [fieldValue];
|
|
|
|
//console.log(util.inspect(d, { depth: null }));
|
|
//process.exit(1);
|
|
|
|
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(d),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
console.error('Error notifying sync completion:', response.statusText);
|
|
process.exit(1);
|
|
}
|
|
|
|
const data = await response.json();
|
|
total = data.results[0].totalResults;
|
|
|
|
//console.log(data);
|
|
|
|
const poolConnection = mysql.createPool({
|
|
uri: process.env.DATABASE_URL,
|
|
});
|
|
|
|
const db = drizzle(poolConnection, { schema, mode: 'default' });
|
|
|
|
for (const item of data.results[0].results) {
|
|
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
|
|
|
|
await db.insert(schema.cards).values({
|
|
productId: item.productId,
|
|
productName: item.productName,
|
|
rarityName: item.rarityName,
|
|
productLineName: item.productLineName,
|
|
productLineUrlName: item.productLineUrlName,
|
|
productStatusId: item.productStatusId,
|
|
productTypeId: item.productTypeId,
|
|
productUrlName: item.productUrlName,
|
|
setId: item.setId,
|
|
shippingCategoryId: item.shippingCategoryId,
|
|
sealed: item.sealed,
|
|
sellerListable: item.sellerListable,
|
|
foilOnly: item.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: item.customAttributes.energyType?.[0] || null,
|
|
flavorText: item.customAttributes.flavorText || null,
|
|
hp: item.customAttributes.hp || 0,
|
|
number: item.customAttributes.number || '',
|
|
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: item.lowestPrice,
|
|
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
|
marketPrice: item.marketPrice,
|
|
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
|
medianPrice: item.medianPrice,
|
|
totalListings: item.totalListings,
|
|
}).onDuplicateKeyUpdate({
|
|
set: {
|
|
productName: item.productName,
|
|
rarityName: item.rarityName,
|
|
productLineName: item.productLineName,
|
|
productLineUrlName: item.productLineUrlName,
|
|
productStatusId: item.productStatusId,
|
|
productTypeId: item.productTypeId,
|
|
productUrlName: item.productUrlName,
|
|
setId: item.setId,
|
|
shippingCategoryId: item.shippingCategoryId,
|
|
sealed: item.sealed,
|
|
sellerListable: item.sellerListable,
|
|
foilOnly: item.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: item.customAttributes.energyType?.[0] || null,
|
|
flavorText: item.customAttributes.flavorText || null,
|
|
hp: item.customAttributes.hp || 0,
|
|
number: item.customAttributes.number || '',
|
|
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: item.lowestPrice,
|
|
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
|
marketPrice: item.marketPrice,
|
|
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
|
medianPrice: item.medianPrice,
|
|
totalListings: item.totalListings,
|
|
},
|
|
});
|
|
|
|
// before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls.
|
|
const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId));
|
|
const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000);
|
|
if (hasRecentSku) {
|
|
console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data'));
|
|
await sleep(100);
|
|
continue;
|
|
}
|
|
|
|
// Get product detail
|
|
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
|
|
method: 'GET',
|
|
});
|
|
|
|
if (!detailResponse.ok) {
|
|
console.error('Error fetching product details:', detailResponse.statusText);
|
|
process.exit(1);
|
|
}
|
|
|
|
const detailData = await detailResponse.json();
|
|
|
|
await db.insert(schema.sets).values({
|
|
setId: detailData.setId,
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
}).onDuplicateKeyUpdate({
|
|
set: {
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
},
|
|
});
|
|
|
|
// skus are...
|
|
const skuArray = detailData.skus.map((sku: any) => sku.sku);
|
|
//console.log(detailData.skus);
|
|
//console.log(skuArray);
|
|
// get pricing for skus
|
|
const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify({ skuIds: skuArray }),
|
|
});
|
|
|
|
if (!skuResponse.ok) {
|
|
console.error('Error fetching SKU pricing:', skuResponse.statusText);
|
|
process.exit(1);
|
|
}
|
|
|
|
const skuData = await skuResponse.json();
|
|
|
|
let skuMap = new Map();
|
|
for (const skuItem of skuData) {
|
|
skuMap.set(skuItem.skuId, skuItem);
|
|
}
|
|
|
|
for (const skuItem of detailData.skus) {
|
|
const pricing = skuMap.get(skuItem.sku);
|
|
//console.log(pricing);
|
|
|
|
await db.insert(schema.skus).values({
|
|
skuId: skuItem.sku,
|
|
productId: detailData.productId,
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
|
|
highestPrice: pricing?.highestPrice || null,
|
|
lowestPrice: pricing?.lowestPrice || null,
|
|
marketPrice: pricing?.marketPrice || null,
|
|
priceCount: pricing?.priceCount || 0,
|
|
}).onDuplicateKeyUpdate({
|
|
set: {
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
|
|
highestPrice: pricing?.highestPrice || null,
|
|
lowestPrice: pricing?.lowestPrice || null,
|
|
marketPrice: pricing?.marketPrice || null,
|
|
priceCount: pricing?.priceCount || 0,
|
|
},
|
|
});
|
|
}
|
|
|
|
// get image if it doesn't already exist
|
|
const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
|
|
if (!await fileExists(imagePath)) {
|
|
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
|
|
if (imageResponse.ok) {
|
|
const buffer = await imageResponse.arrayBuffer();
|
|
await fs.writeFile(imagePath, Buffer.from(buffer));
|
|
} else {
|
|
console.error('Error fetching product image:', imageResponse.statusText);
|
|
}
|
|
}
|
|
|
|
// be nice to the API and not send too many requests in a short time
|
|
await sleep(100);
|
|
|
|
}
|
|
|
|
|
|
await poolConnection.end();
|
|
|
|
start += size;
|
|
}
|
|
}
|
|
|
|
|
|
syncTcgplayer();
|