290 lines
10 KiB
TypeScript
290 lines
10 KiB
TypeScript
import 'dotenv/config';
|
|
import * as schema from '../src/db/schema.ts';
|
|
import { db, poolConnection } from '../src/db/index.ts';
|
|
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import chalk from 'chalk';
|
|
//import util from 'util';
|
|
|
|
|
|
async function syncTcgplayer() {
|
|
|
|
const productLines = [ "pokemon", "pokemon-japan" ];
|
|
|
|
// work from the available sets within the product line
|
|
for (const productLine of productLines) {
|
|
const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
|
|
|
|
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: {'Content-Type': 'application/json',},
|
|
body: JSON.stringify(d),
|
|
});
|
|
if (!response.ok) {
|
|
console.error('Error notifying sync completion:', response.statusText);
|
|
process.exit(1);
|
|
}
|
|
const data = await response.json();
|
|
|
|
const setNames = data.results[0].aggregations.setName;
|
|
for (const setName of setNames) {
|
|
console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
|
|
await syncProductLine(productLine, "setName", setName.urlValue);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
|
|
}
|
|
|
|
function sleep(ms: number) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function cleanProductName(name: string): string {
|
|
// remove TCGPlayer crap
|
|
name = name.replace(/ - .*$/, '');
|
|
name = name.replace(/ \[.*\]/, '');
|
|
name = name.replace(/ \(.*\)/, '');
|
|
return name.trim();
|
|
}
|
|
|
|
async function fileExists(path: string): Promise<boolean> {
|
|
try {
|
|
await fs.access(path);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function getNumberOrNull(value: any): number | null {
|
|
const number = Number(value); // Attempt to convert the value to a number
|
|
if (Number.isNaN(number)) {
|
|
return null; // Return null if the result is NaN
|
|
}
|
|
return number; // Otherwise, return the number
|
|
}
|
|
|
|
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
|
|
let start = 0;
|
|
let size = 50;
|
|
let total = 1000000;
|
|
|
|
while (start < total) {
|
|
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
|
|
|
const d = {
|
|
"algorithm":"sales_dismax",
|
|
"from":start,
|
|
"size":size,
|
|
"filters":{
|
|
"term":{"productLineName":[productLine], [field]:[fieldValue]} ,
|
|
"range":{},
|
|
"match":{}
|
|
},
|
|
"listingSearch":{
|
|
"context":{"cart":{}},
|
|
"filters":{"term":{
|
|
"sellerStatus":"Live",
|
|
"channelId":0
|
|
},
|
|
"range":{
|
|
"quantity":{"gte":1}
|
|
},
|
|
"exclude":{"channelExclusion":0}
|
|
}
|
|
},
|
|
"context":{
|
|
"cart":{},
|
|
"shippingCountry":"US",
|
|
"userProfile":{}
|
|
},
|
|
"settings":{
|
|
"useFuzzySearch":false,
|
|
"didYouMean":{}
|
|
},
|
|
"sort":{}
|
|
};
|
|
|
|
//console.log(util.inspect(d, { depth: null }));
|
|
//process.exit(1);
|
|
|
|
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(d),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
console.error('Error notifying sync completion:', response.statusText);
|
|
process.exit(1);
|
|
}
|
|
|
|
const data = await response.json();
|
|
total = data.results[0].totalResults;
|
|
|
|
for (const item of data.results[0].results) {
|
|
|
|
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
|
|
if (allProductIds.has(item.productId)) {
|
|
continue;
|
|
}
|
|
|
|
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
|
|
|
|
// Get product detail
|
|
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
|
|
method: 'GET',
|
|
});
|
|
if (!detailResponse.ok) {
|
|
console.error('Error fetching product details:', detailResponse.statusText);
|
|
process.exit(1);
|
|
}
|
|
const detailData = await detailResponse.json();
|
|
|
|
|
|
await db.insert(schema.cards).values({
|
|
productId: item.productId,
|
|
originalProductName: item.productName,
|
|
productName: cleanProductName(item.productName),
|
|
rarityName: item.rarityName,
|
|
productLineName: item.productLineName,
|
|
productLineUrlName: item.productLineUrlName,
|
|
productStatusId: item.productStatusId,
|
|
productTypeId: item.productTypeId,
|
|
productUrlName: item.productUrlName,
|
|
setId: item.setId,
|
|
shippingCategoryId: item.shippingCategoryId,
|
|
sealed: item.sealed,
|
|
sellerListable: item.sellerListable,
|
|
foilOnly: item.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: item.customAttributes.energyType?.[0] || null,
|
|
flavorText: item.customAttributes.flavorText || null,
|
|
hp: getNumberOrNull(item.customAttributes.hp),
|
|
number: item.customAttributes.number || '',
|
|
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: item.lowestPrice,
|
|
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
|
marketPrice: item.marketPrice,
|
|
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
|
medianPrice: item.medianPrice,
|
|
totalListings: item.totalListings,
|
|
Artist: detailData.formattedAttributes.Artist || null,
|
|
}).onDuplicateKeyUpdate({
|
|
set: {
|
|
originalProductName: item.productName,
|
|
productName: cleanProductName(item.productName),
|
|
rarityName: item.rarityName,
|
|
productLineName: item.productLineName,
|
|
productLineUrlName: item.productLineUrlName,
|
|
productStatusId: item.productStatusId,
|
|
productTypeId: item.productTypeId,
|
|
productUrlName: item.productUrlName,
|
|
setId: item.setId,
|
|
shippingCategoryId: item.shippingCategoryId,
|
|
sealed: item.sealed,
|
|
sellerListable: item.sellerListable,
|
|
foilOnly: item.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: item.customAttributes.energyType?.[0] || null,
|
|
flavorText: item.customAttributes.flavorText || null,
|
|
hp: getNumberOrNull(item.customAttributes.hp),
|
|
number: item.customAttributes.number || '',
|
|
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: item.lowestPrice,
|
|
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
|
marketPrice: item.marketPrice,
|
|
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
|
medianPrice: item.medianPrice,
|
|
totalListings: item.totalListings,
|
|
Artist: detailData.formattedAttributes.Artist || null,
|
|
},
|
|
});
|
|
|
|
|
|
// set is...
|
|
await db.insert(schema.sets).values({
|
|
setId: detailData.setId,
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
}).onDuplicateKeyUpdate({
|
|
set: {
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
},
|
|
});
|
|
|
|
// skus are...
|
|
for (const skuItem of detailData.skus) {
|
|
|
|
await db.insert(schema.skus).values({
|
|
skuId: skuItem.sku,
|
|
productId: detailData.productId,
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
}).onDuplicateKeyUpdate({
|
|
set: {
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
},
|
|
});
|
|
}
|
|
|
|
// get image if it doesn't already exist
|
|
const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
|
|
if (!await fileExists(imagePath)) {
|
|
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
|
|
if (imageResponse.ok) {
|
|
const buffer = await imageResponse.arrayBuffer();
|
|
await fs.writeFile(imagePath, Buffer.from(buffer));
|
|
} else {
|
|
console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
|
|
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
|
|
}
|
|
}
|
|
|
|
// be nice to the API and not send too many requests in a short time
|
|
await sleep(100);
|
|
|
|
}
|
|
|
|
start += size;
|
|
}
|
|
}
|
|
|
|
// clear the log file
|
|
await fs.rm('missing_images.log', { force: true });
|
|
|
|
const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
|
|
|
|
await syncTcgplayer();
|
|
await poolConnection.end();
|