324 lines
12 KiB
TypeScript
324 lines
12 KiB
TypeScript
import 'dotenv/config';
|
|
import * as schema from '../src/db/schema.ts';
|
|
import { db, ClosePool, type DBInstance } from '../src/db/index.ts';
|
|
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from 'node:url';
|
|
import * as helper from './pokemon-helper.ts';
|
|
|
|
export type Logger = (msg: string) => void;
|
|
const consoleLogger: Logger = (m) => console.log(m);
|
|
|
|
export type RunImportOptions = {
|
|
sets?: string[];
|
|
log?: Logger;
|
|
runUpdateVariants?: boolean;
|
|
runCardUpsert?: boolean;
|
|
};
|
|
|
|
|
|
const syncProductLine = async (
|
|
database: DBInstance,
|
|
productLine: string,
|
|
field: string,
|
|
fieldValue: string,
|
|
allProductIds: Set<number>,
|
|
log: Logger,
|
|
) => {
|
|
let start = 0;
|
|
const size = 50;
|
|
let total = 1000000;
|
|
|
|
while (start < total) {
|
|
log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
|
|
|
const d = {
|
|
"algorithm": "sales_dismax",
|
|
"from": start,
|
|
"size": size,
|
|
"filters": {
|
|
"term": { "productLineName": [productLine], [field]: [fieldValue] },
|
|
"range": {},
|
|
"match": {},
|
|
},
|
|
"listingSearch": {
|
|
"context": { "cart": {} },
|
|
"filters": {
|
|
"term": { "sellerStatus": "Live", "channelId": 0 },
|
|
"range": { "quantity": { "gte": 1 } },
|
|
"exclude": { "channelExclusion": 0 },
|
|
},
|
|
},
|
|
"context": { "cart": {}, "shippingCountry": "US", "userProfile": {} },
|
|
"settings": { "useFuzzySearch": false, "didYouMean": {} },
|
|
"sort": {},
|
|
};
|
|
|
|
let response: Response | null = null;
|
|
let lastError: unknown = null;
|
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
try {
|
|
const r = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(d),
|
|
});
|
|
if (r.ok) {
|
|
response = r;
|
|
break;
|
|
}
|
|
lastError = new Error(`TCGPlayer search request failed: ${r.status} ${r.statusText}`);
|
|
} catch (err) {
|
|
lastError = err;
|
|
}
|
|
if (attempt < 3) {
|
|
log(` retry ${attempt}/2 for search request in 5s...`);
|
|
await helper.Sleep(5000);
|
|
}
|
|
}
|
|
if (!response) {
|
|
throw lastError instanceof Error
|
|
? lastError
|
|
: new Error(`TCGPlayer search request failed: ${String(lastError)}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
total = data.results[0].totalResults;
|
|
|
|
for (const item of data.results[0].results) {
|
|
|
|
if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
|
|
continue;
|
|
}
|
|
|
|
log(` - ${item.productName} (ID: ${item.productId})`);
|
|
|
|
let detailResponse: Response | null = null;
|
|
let lastError: unknown = null;
|
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
try {
|
|
const r = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
|
|
method: 'GET',
|
|
});
|
|
if (r.ok) {
|
|
detailResponse = r;
|
|
break;
|
|
}
|
|
lastError = new Error(`Error fetching product details for ${item.productId}: ${r.statusText}`);
|
|
} catch (err) {
|
|
lastError = err;
|
|
}
|
|
if (attempt < 3) {
|
|
log(` retry ${attempt}/2 for product ${item.productId} in 5s...`);
|
|
await helper.Sleep(5000);
|
|
}
|
|
}
|
|
if (!detailResponse) {
|
|
throw lastError instanceof Error
|
|
? lastError
|
|
: new Error(`Error fetching product details for ${item.productId}: ${String(lastError)}`);
|
|
}
|
|
const detailData = await detailResponse.json();
|
|
|
|
|
|
await database.insert(schema.tcgcards).values({
|
|
productId: item.productId,
|
|
productName: detailData.productName,
|
|
rarityName: item.rarityName,
|
|
productLineName: detailData.productLineName,
|
|
productLineUrlName: detailData.productLineUrlName,
|
|
productStatusId: detailData.productStatusId,
|
|
productTypeId: detailData.productTypeId,
|
|
productUrlName: detailData.productUrlName,
|
|
setId: detailData.setId,
|
|
shippingCategoryId: detailData.shippingCategoryId,
|
|
sealed: detailData.sealed,
|
|
sellerListable: detailData.sellerListable,
|
|
foilOnly: detailData.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: detailData.customAttributes.energyType?.[0] || null,
|
|
flavorText: detailData.customAttributes.flavorText || null,
|
|
hp: helper.GetNumberOrNull(item.customAttributes.hp),
|
|
number: detailData.customAttributes.number || '',
|
|
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: detailData.lowestPrice,
|
|
lowestPriceWithShipping: detailData.lowestPriceWithShipping,
|
|
marketPrice: detailData.marketPrice,
|
|
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
|
|
medianPrice: detailData.medianPrice,
|
|
totalListings: item.totalListings,
|
|
artist: detailData.formattedAttributes.Artist || null,
|
|
}).onConflictDoUpdate({
|
|
target: schema.tcgcards.productId,
|
|
set: {
|
|
productName: detailData.productName,
|
|
rarityName: item.rarityName,
|
|
productLineName: detailData.productLineName,
|
|
productLineUrlName: detailData.productLineUrlName,
|
|
productStatusId: detailData.productStatusId,
|
|
productTypeId: detailData.productTypeId,
|
|
productUrlName: detailData.productUrlName,
|
|
setId: detailData.setId,
|
|
shippingCategoryId: detailData.shippingCategoryId,
|
|
sealed: detailData.sealed,
|
|
sellerListable: detailData.sellerListable,
|
|
foilOnly: detailData.foilOnly,
|
|
attack1: item.customAttributes.attack1 || null,
|
|
attack2: item.customAttributes.attack2 || null,
|
|
attack3: item.customAttributes.attack3 || null,
|
|
attack4: item.customAttributes.attack4 || null,
|
|
cardType: item.customAttributes.cardType?.[0] || null,
|
|
cardTypeB: item.customAttributes.cardTypeB || null,
|
|
energyType: detailData.customAttributes.energyType?.[0] || null,
|
|
flavorText: detailData.customAttributes.flavorText || null,
|
|
hp: helper.GetNumberOrNull(item.customAttributes.hp),
|
|
number: detailData.customAttributes.number || '',
|
|
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
|
|
resistance: item.customAttributes.resistance || null,
|
|
retreatCost: item.customAttributes.retreatCost || null,
|
|
stage: item.customAttributes.stage || null,
|
|
weakness: item.customAttributes.weakness || null,
|
|
lowestPrice: detailData.lowestPrice,
|
|
lowestPriceWithShipping: detailData.lowestPriceWithShipping,
|
|
marketPrice: detailData.marketPrice,
|
|
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
|
|
medianPrice: detailData.medianPrice,
|
|
totalListings: item.totalListings,
|
|
artist: detailData.formattedAttributes.Artist || null,
|
|
},
|
|
});
|
|
|
|
await database.insert(schema.sets).values({
|
|
setId: detailData.setId,
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
}).onConflictDoUpdate({
|
|
target: schema.sets.setId,
|
|
set: {
|
|
setCode: detailData.setCode,
|
|
setName: detailData.setName,
|
|
setUrlName: detailData.setUrlName,
|
|
},
|
|
});
|
|
|
|
for (const skuItem of detailData.skus) {
|
|
await database.insert(schema.skus).values({
|
|
skuId: skuItem.sku,
|
|
productId: detailData.productId,
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
}).onConflictDoUpdate({
|
|
target: schema.skus.skuId,
|
|
set: {
|
|
condition: skuItem.condition,
|
|
language: skuItem.language,
|
|
variant: skuItem.variant,
|
|
},
|
|
});
|
|
}
|
|
|
|
const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
|
|
if (!await helper.FileExists(imagePath)) {
|
|
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
|
|
if (imageResponse.ok) {
|
|
const buffer = await imageResponse.arrayBuffer();
|
|
await fs.writeFile(imagePath, Buffer.from(buffer));
|
|
} else {
|
|
log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`);
|
|
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
|
|
}
|
|
}
|
|
|
|
await helper.Sleep(300);
|
|
}
|
|
|
|
start += size;
|
|
}
|
|
};
|
|
|
|
|
|
const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set<number>, log: Logger) => {
|
|
const productLines = ["pokemon", "pokemon-japan"];
|
|
|
|
for (const productLine of productLines) {
|
|
const d = {
|
|
"algorithm": "sales_dismax", "from": 0, "size": 1,
|
|
"filters": { "term": { "productLineName": [productLine] } },
|
|
"settings": { "useFuzzySearch": false },
|
|
};
|
|
|
|
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(d),
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`);
|
|
}
|
|
const data = await response.json();
|
|
|
|
const setNames = data.results[0].aggregations.setName;
|
|
for (const setName of setNames) {
|
|
let processSet = true;
|
|
if (cardSets.length > 0) {
|
|
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
|
|
}
|
|
if (processSet) {
|
|
log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`);
|
|
await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log);
|
|
}
|
|
}
|
|
}
|
|
|
|
log('All TCGPlayer data synchronized successfully!');
|
|
};
|
|
|
|
|
|
export const runImport = async (opts: RunImportOptions = {}) => {
|
|
const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts;
|
|
|
|
await fs.rm('missing_images.log', { force: true });
|
|
|
|
// When no set filter is provided, skip productIds already in the cards table
|
|
// (matches the CLI script's "no args" behavior).
|
|
const allProductIds = sets.length === 0
|
|
? new Set<number>(
|
|
await db.select({ productId: schema.cards.productId }).from(schema.cards)
|
|
.then(rows => rows.map(row => row.productId))
|
|
)
|
|
: new Set<number>();
|
|
|
|
await syncTcgplayer(db, sets, allProductIds, log);
|
|
|
|
if (runUpdateVariants) {
|
|
log('Updating card variants...');
|
|
await helper.UpdateVariants(db, log);
|
|
}
|
|
|
|
if (runCardUpsert) {
|
|
log('Reindexing "cards" collection...');
|
|
await helper.upsertCardCollection(db, log);
|
|
}
|
|
};
|
|
|
|
|
|
// CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage.
|
|
const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
|
if (isCli) {
|
|
const args = process.argv.slice(2);
|
|
await runImport({ sets: args });
|
|
await ClosePool();
|
|
}
|