tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[feat] add Artist field, and name cleanup

Browse Source
This commit is contained in:
Thad Miller
2026-02-19 16:04:34 -05:00
parent 493d0a72fa
commit 81b223ae65
5 changed files with 54 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
scripts/preload-tcgplayer.ts
View File

@@ -1,10 +1,9 @@
import 'dotenv/config';
import { drizzle } from 'drizzle-orm/mysql2';
import mysql from 'mysql2/promise';
import * as schema from '../src/db/schema.ts';
import { db, poolConnection } from '../src/db/index.ts';
import fs from "node:fs/promises";
import path from "node:path";
import { eq } from 'drizzle-orm';
import chalk from 'chalk';
//import util from 'util';
@@ -12,8 +11,12 @@ import chalk from 'chalk';
async function syncTcgplayer() {
const productLines = [
{ name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
{ name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }
{ name: "pokemon", rarityName: ["Common", "Uncommon", "Promo", "Rare", "Ultra Rare", "Holo Rare", "Code Card", "Secret Rare",
"Illustration Rare", "Double Rare", "Shiny Holo Rare", "Special Illustration Rare", "Classic Collection", "Shiny Rare",
"Hyper Rare", "Unconfirmed", "ACE SPEC Rare", "Prism Rare", "Radiant Rare", "Rare BREAK", "Rare Ace", "Shiny Ultra Rare",
"Amazing Rare", "Mega Attack Rare", "Mega Hyper Rare", "Black White Rare"
] },
{ name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
];
for (const productLine of productLines) {
@@ -21,7 +24,7 @@ async function syncTcgplayer() {
if (key === "name") continue;
for (const value of values) {
console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
await syncProductLineEnergyType(productLine.name, key, value);
await syncProductLine(productLine.name, key, value);
}
}
}
@@ -33,6 +36,14 @@ function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function cleanProductName(name: string): string {
// remove TCGPlayer crap
name = name.replace(/ - .*$/, '');
name = name.replace(/ \[.*\]/, '');
name = name.replace(/ \(.*\)/, '');
return name.trim();
}
async function fileExists(path: string): Promise<boolean> {
try {
await fs.access(path);
@@ -42,7 +53,7 @@ async function fileExists(path: string): Promise<boolean> {
}
}
async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) {
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
let start = 0;
let size = 50;
let total = 1000000;
@@ -50,13 +61,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
while (start < total) {
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
let d = {
const d = {
"algorithm":"sales_dismax",
"from":start,
"size":size,
"filters":{
"term":{"productLineName":[productLine]},
"term":{"productLineName":[productLine], [field]:[fieldValue]} ,
"range":{},
"match":{}
},
@@ -83,7 +93,6 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
},
"sort":{}
};
d.filters.term[field] = [fieldValue];
//console.log(util.inspect(d, { depth: null }));
//process.exit(1);
@@ -104,20 +113,24 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
const data = await response.json();
total = data.results[0].totalResults;
//console.log(data);
const poolConnection = mysql.createPool({
uri: process.env.DATABASE_URL,
});
const db = drizzle(poolConnection, { schema, mode: 'default' });
for (const item of data.results[0].results) {
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
// Get product detail
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
method: 'GET',
});
if (!detailResponse.ok) {
console.error('Error fetching product details:', detailResponse.statusText);
process.exit(1);
}
const detailData = await detailResponse.json();
await db.insert(schema.cards).values({
productId: item.productId,
productName: item.productName,
originalProductName: item.productName,
productName: cleanProductName(item.productName),
rarityName: item.rarityName,
productLineName: item.productLineName,
productLineUrlName: item.productLineUrlName,
@@ -150,9 +163,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
maxFulfillableQuantity: item.maxFulfillableQuantity,
medianPrice: item.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
}).onDuplicateKeyUpdate({
set: {
productName: item.productName,
originalProductName: item.productName,
productName: cleanProductName(item.productName),
rarityName: item.rarityName,
productLineName: item.productLineName,
productLineUrlName: item.productLineUrlName,
@@ -185,30 +200,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
maxFulfillableQuantity: item.maxFulfillableQuantity,
medianPrice: item.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
},
});
// before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls.
const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId));
const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000);
if (hasRecentSku) {
console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data'));
await sleep(100);
continue;
}
// Get product detail
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
method: 'GET',
});
if (!detailResponse.ok) {
console.error('Error fetching product details:', detailResponse.statusText);
process.exit(1);
}
const detailData = await detailResponse.json();
// set is...
await db.insert(schema.sets).values({
setId: detailData.setId,
setCode: detailData.setCode,
@@ -223,33 +220,7 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
});
// skus are...
const skuArray = detailData.skus.map((sku: any) => sku.sku);
//console.log(detailData.skus);
//console.log(skuArray);
// get pricing for skus
const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ skuIds: skuArray }),
});
if (!skuResponse.ok) {
console.error('Error fetching SKU pricing:', skuResponse.statusText);
process.exit(1);
}
const skuData = await skuResponse.json();
let skuMap = new Map();
for (const skuItem of skuData) {
skuMap.set(skuItem.skuId, skuItem);
}
for (const skuItem of detailData.skus) {
const pricing = skuMap.get(skuItem.sku);
//console.log(pricing);
await db.insert(schema.skus).values({
skuId: skuItem.sku,
@@ -257,21 +228,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
condition: skuItem.condition,
language: skuItem.language,
variant: skuItem.variant,
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
highestPrice: pricing?.highestPrice || null,
lowestPrice: pricing?.lowestPrice || null,
marketPrice: pricing?.marketPrice || null,
priceCount: pricing?.priceCount || 0,
}).onDuplicateKeyUpdate({
set: {
condition: skuItem.condition,
language: skuItem.language,
variant: skuItem.variant,
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
highestPrice: pricing?.highestPrice || null,
lowestPrice: pricing?.lowestPrice || null,
marketPrice: pricing?.marketPrice || null,
priceCount: pricing?.priceCount || 0,
},
});
}
@@ -284,7 +245,8 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
const buffer = await imageResponse.arrayBuffer();
await fs.writeFile(imagePath, Buffer.from(buffer));
} else {
console.error('Error fetching product image:', imageResponse.statusText);
console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
}
}
@@ -293,12 +255,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
}
await poolConnection.end();
start += size;
}
}
// clear the log file
await fs.rm('missing_images.log', { force: true });
syncTcgplayer();
await syncTcgplayer();
await poolConnection.end();