286 lines
11 KiB
TypeScript
286 lines
11 KiB
TypeScript
import 'dotenv/config';
|
|
import * as schema from '../src/db/schema.ts';
|
|
import { db, ClosePool } from '../src/db/index.ts';
|
|
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import chalk from 'chalk';
|
|
import * as helper from './pokemon-helper.ts';
|
|
//import util from 'util';
|
|
|
|
|
|
async function syncTcgplayer(cardSets:string[] = []) {
|
|
|
|
const productLines = [ "pokemon", "pokemon-japan" ];
|
|
|
|
// work from the available sets within the product line
|
|
for (const productLine of productLines) {
|
|
const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
|
|
|
|
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: {'Content-Type': 'application/json',},
|
|
body: JSON.stringify(d),
|
|
});
|
|
if (!response.ok) {
|
|
console.error('Error notifying sync completion:', response.statusText);
|
|
process.exit(1);
|
|
}
|
|
const data = await response.json();
|
|
|
|
const setNames = data.results[0].aggregations.setName;
|
|
for (const setName of setNames) {
|
|
let processSet = true;
|
|
if (cardSets.length > 0) {
|
|
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
|
|
}
|
|
if (processSet) {
|
|
console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
|
|
await syncProductLine(productLine, "setName", setName.urlValue);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
|
|
}
|
|
|
|
|
|
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
|
|
let start = 0;
|
|
let size = 50;
|
|
let total = 1000000;
|
|
|
|
while (start < total) {
|
|
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
|
|
|
const d = {
|
|
"algorithm":"sales_dismax",
|
|
"from":start,
|
|
"size":size,
|
|
"filters":{
|
|
"term":{"productLineName":[productLine], [field]:[fieldValue]} ,
|
|
"range":{},
|
|
"match":{}
|
|
},
|
|
"listingSearch":{
|
|
"context":{"cart":{}},
|
|
"filters":{"term":{
|
|
"sellerStatus":"Live",
|
|
"channelId":0
|
|
},
|
|
"range":{
|
|
"quantity":{"gte":1}
|
|
},
|
|
"exclude":{"channelExclusion":0}
|
|
}
|
|
},
|
|
"context":{
|
|
"cart":{},
|
|
"shippingCountry":"US",
|
|
"userProfile":{}
|
|
},
|
|
"settings":{
|
|
"useFuzzySearch":false,
|
|
"didYouMean":{}
|
|
},
|
|
"sort":{}
|
|
};
|
|
|
|
//console.log(util.inspect(d, { depth: null }));
|
|
//process.exit(1);
|
|
|
|
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(d),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
console.error('Error notifying sync completion:', response.statusText);
|
|
process.exit(1);
|
|
}
|
|
|
|
const data = await response.json();
|
|
total = data.results[0].totalResults;
|
|
|
|
for (const item of data.results[0].results) {
|
|
|
|
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
|
|
if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
|
|
continue;
|
|
}
|
|
|
|
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
|
|
|
|
// Get product detail
|
|
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
|
|
method: 'GET',
|
|
});
|
|
if (!detailResponse.ok) {
|
|
console.error('Error fetching product details:', detailResponse.statusText);
|
|
process.exit(1);
|
|
}
|
|
const detailData = await detailResponse.json();
|
|
|
|
|
|
await db.insert(schema.tcgcards).values({
|
|
productId: item.productId,
|
|
productName: detailData.productName,
|
|
//productName: cleanProductName(item.productName),
|
|
rarityName: item.rarityName,
|
|
productLineName: detailData.productLineName,
|
|
productLineUrlName: detailData.productLineUrlName,
|
|
productStatusId: detailData.productStatusId,
|
|
productTypeId: detailData.productTypeId,
|
|
productUrlName: detailData.productUrlName,
|
|
setId: detailData.setId,
|
|
shippingCategoryId: detailData.shippingCategoryId,
|
|
sealed: detailData.sealed,
|
|
sellerListable: detailData.sellerListable,
|
|
foilOnly: detailData.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: detailData.customAttributes.energyType?.[0] || null,
|
|
flavorText: detailData.customAttributes.flavorText || null,
|
|
hp: helper.GetNumberOrNull(item.customAttributes.hp),
|
|
number: detailData.customAttributes.number || '',
|
|
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: detailData.lowestPrice,
|
|
lowestPriceWithShipping: detailData.lowestPriceWithShipping,
|
|
marketPrice: detailData.marketPrice,
|
|
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
|
|
medianPrice: detailData.medianPrice,
|
|
totalListings: item.totalListings,
|
|
artist: detailData.formattedAttributes.Artist || null,
|
|
}).onConflictDoUpdate({
|
|
target: schema.tcgcards.productId,
|
|
set: {
|
|
productName: detailData.productName,
|
|
//productName: cleanProductName(item.productName),
|
|
rarityName: item.rarityName,
|
|
productLineName: detailData.productLineName,
|
|
productLineUrlName: detailData.productLineUrlName,
|
|
productStatusId: detailData.productStatusId,
|
|
productTypeId: detailData.productTypeId,
|
|
productUrlName: detailData.productUrlName,
|
|
setId: detailData.setId,
|
|
shippingCategoryId: detailData.shippingCategoryId,
|
|
sealed: detailData.sealed,
|
|
sellerListable: detailData.sellerListable,
|
|
foilOnly: detailData.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: detailData.customAttributes.energyType?.[0] || null,
|
|
flavorText: detailData.customAttributes.flavorText || null,
|
|
hp: helper.GetNumberOrNull(item.customAttributes.hp),
|
|
number: detailData.customAttributes.number || '',
|
|
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: detailData.lowestPrice,
|
|
lowestPriceWithShipping: detailData.lowestPriceWithShipping,
|
|
marketPrice: detailData.marketPrice,
|
|
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
|
|
medianPrice: detailData.medianPrice,
|
|
totalListings: item.totalListings,
|
|
artist: detailData.formattedAttributes.Artist || null,
|
|
},
|
|
});
|
|
|
|
// console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
|
|
// console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
|
|
// console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
|
|
// set is...
|
|
await db.insert(schema.sets).values({
|
|
setId: detailData.setId,
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
}).onConflictDoUpdate({
|
|
target: schema.sets.setId,
|
|
set: {
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
},
|
|
});
|
|
|
|
// skus are...
|
|
for (const skuItem of detailData.skus) {
|
|
|
|
await db.insert(schema.skus).values({
|
|
skuId: skuItem.sku,
|
|
productId: detailData.productId,
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
}).onConflictDoUpdate({
|
|
target: schema.skus.skuId,
|
|
set: {
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
},
|
|
});
|
|
}
|
|
|
|
// get image if it doesn't already exist
|
|
const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
|
|
if (!await helper.FileExists(imagePath)) {
|
|
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
|
|
if (imageResponse.ok) {
|
|
const buffer = await imageResponse.arrayBuffer();
|
|
await fs.writeFile(imagePath, Buffer.from(buffer));
|
|
} else {
|
|
console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
|
|
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
|
|
}
|
|
}
|
|
|
|
// be nice to the API and not send too many requests in a short time
|
|
await helper.Sleep(300);
|
|
|
|
}
|
|
|
|
start += size;
|
|
}
|
|
}
|
|
|
|
// clear the log file
|
|
await fs.rm('missing_images.log', { force: true });
|
|
let allProductIds = new Set();
|
|
|
|
const args = process.argv.slice(2);
|
|
if (args.length === 0) {
|
|
allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
|
|
await syncTcgplayer();
|
|
}
|
|
else {
|
|
await syncTcgplayer(args);
|
|
}
|
|
|
|
// update the card table with new/updated variants
|
|
await helper.UpdateVariants(db);
|
|
|
|
// index the card updates
|
|
await helper.upsertCardCollection(db);
|
|
|
|
await ClosePool();
|