tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[feat] add Artist field, and name cleanup

Browse Source
This commit is contained in:
Thad Miller
2026-02-19 16:04:34 -05:00
parent 493d0a72fa
commit 81b223ae65
5 changed files with 54 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -28,3 +28,6 @@ public/cards/*
# anything test
test.*
# any logs
*.log

6
scripts/preload-searchindex.ts
View File

@@ -8,9 +8,9 @@ async function createCollection(client: Client) {
// Delete the collection if it already exists to ensure a clean slate
try {
const response = await client.collections('cards').delete();
console.log(`Collection "cards" deleted successfully:`, response);
//console.log(`Collection "cards" deleted successfully:`, response);
} catch (error) {
console.error(`Error deleting collection "cards":`, error);
//console.error(`Error deleting collection "cards":`, error);
}
// Create the collection with the specified schema
@@ -30,6 +30,7 @@ async function createCollection(client: Client) {
{ name: 'cardType', type: 'string', facet: true },
{ name: 'energyType', type: 'string', facet: true },
{ name: 'number', type: 'string' },
{ name: 'Artist', type: 'string' },
],
default_sorting_field: 'productId',
});
@@ -59,6 +60,7 @@ async function preloadSearchIndex() {
cardType: card.cardType || "",
energyType: card.energyType || "",
number: card.number,
Artist: card.Artist || "",
})), { action: 'upsert' });
console.log(chalk.green('Search index preloaded with Pokémon cards.'));

250
scripts/preload-tcgplayer.ts
View File

@@ -1,10 +1,9 @@
import 'dotenv/config';
import { drizzle } from 'drizzle-orm/mysql2';
import mysql from 'mysql2/promise';
import * as schema from '../src/db/schema.ts';
import { db, poolConnection } from '../src/db/index.ts';
import fs from "node:fs/promises";
import path from "node:path";
import { eq } from 'drizzle-orm';
import chalk from 'chalk';
//import util from 'util';
@@ -12,8 +11,12 @@ import chalk from 'chalk';
async function syncTcgplayer() {
const productLines = [
{ name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
{ name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }
{ name: "pokemon", rarityName: ["Common", "Uncommon", "Promo", "Rare", "Ultra Rare", "Holo Rare", "Code Card", "Secret Rare",
"Illustration Rare", "Double Rare", "Shiny Holo Rare", "Special Illustration Rare", "Classic Collection", "Shiny Rare",
"Hyper Rare", "Unconfirmed", "ACE SPEC Rare", "Prism Rare", "Radiant Rare", "Rare BREAK", "Rare Ace", "Shiny Ultra Rare",
"Amazing Rare", "Mega Attack Rare", "Mega Hyper Rare", "Black White Rare"
] },
{ name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
];
for (const productLine of productLines) {
@@ -21,7 +24,7 @@ async function syncTcgplayer() {
if (key === "name") continue;
for (const value of values) {
console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
await syncProductLineEnergyType(productLine.name, key, value);
await syncProductLine(productLine.name, key, value);
}
}
}
@@ -33,6 +36,14 @@ function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function cleanProductName(name: string): string {
// remove TCGPlayer crap
name = name.replace(/ - .*$/, '');
name = name.replace(/ \[.*\]/, '');
name = name.replace(/ \(.*\)/, '');
return name.trim();
}
async function fileExists(path: string): Promise<boolean> {
try {
await fs.access(path);
@@ -42,7 +53,7 @@ async function fileExists(path: string): Promise<boolean> {
}
}
async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) {
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
let start = 0;
let size = 50;
let total = 1000000;
@@ -50,13 +61,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
while (start < total) {
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
let d = {
const d = {
"algorithm":"sales_dismax",
"from":start,
"size":size,
"filters":{
"term":{"productLineName":[productLine]},
"term":{"productLineName":[productLine], [field]:[fieldValue]} ,
"range":{},
"match":{}
},
@@ -83,7 +93,6 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
},
"sort":{}
};
d.filters.term[field] = [fieldValue];
//console.log(util.inspect(d, { depth: null }));
//process.exit(1);
@@ -104,111 +113,99 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
const data = await response.json();
total = data.results[0].totalResults;
//console.log(data);
const poolConnection = mysql.createPool({
uri: process.env.DATABASE_URL,
});
const db = drizzle(poolConnection, { schema, mode: 'default' });
for (const item of data.results[0].results) {
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
await db.insert(schema.cards).values({
productId: item.productId,
productName: item.productName,
rarityName: item.rarityName,
productLineName: item.productLineName,
productLineUrlName: item.productLineUrlName,
productStatusId: item.productStatusId,
productTypeId: item.productTypeId,
productUrlName: item.productUrlName,
setId: item.setId,
shippingCategoryId: item.shippingCategoryId,
sealed: item.sealed,
sellerListable: item.sellerListable,
foilOnly: item.foilOnly,
attack1: item.customAttributes.attack1 || null,
attack2: item.customAttributes.attack2 || null,
attack3: item.customAttributes.attack3 || null,
attack4: item.customAttributes.attack4 || null,
cardType: item.customAttributes.cardType?.[0] || null,
cardTypeB: item.customAttributes.cardTypeB || null,
energyType: item.customAttributes.energyType?.[0] || null,
flavorText: item.customAttributes.flavorText || null,
hp: item.customAttributes.hp || 0,
number: item.customAttributes.number || '',
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null,
retreatCost: item.customAttributes.retreatCost || null,
stage: item.customAttributes.stage || null,
weakness: item.customAttributes.weakness || null,
lowestPrice: item.lowestPrice,
lowestPriceWithShipping: item.lowestPriceWithShipping,
marketPrice: item.marketPrice,
maxFulfillableQuantity: item.maxFulfillableQuantity,
medianPrice: item.medianPrice,
totalListings: item.totalListings,
}).onDuplicateKeyUpdate({
set: {
productName: item.productName,
rarityName: item.rarityName,
productLineName: item.productLineName,
productLineUrlName: item.productLineUrlName,
productStatusId: item.productStatusId,
productTypeId: item.productTypeId,
productUrlName: item.productUrlName,
setId: item.setId,
shippingCategoryId: item.shippingCategoryId,
sealed: item.sealed,
sellerListable: item.sellerListable,
foilOnly: item.foilOnly,
attack1: item.customAttributes.attack1 || null,
attack2: item.customAttributes.attack2 || null,
attack3: item.customAttributes.attack3 || null,
attack4: item.customAttributes.attack4 || null,
cardType: item.customAttributes.cardType?.[0] || null,
cardTypeB: item.customAttributes.cardTypeB || null,
energyType: item.customAttributes.energyType?.[0] || null,
flavorText: item.customAttributes.flavorText || null,
hp: item.customAttributes.hp || 0,
number: item.customAttributes.number || '',
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null,
retreatCost: item.customAttributes.retreatCost || null,
stage: item.customAttributes.stage || null,
weakness: item.customAttributes.weakness || null,
lowestPrice: item.lowestPrice,
lowestPriceWithShipping: item.lowestPriceWithShipping,
marketPrice: item.marketPrice,
maxFulfillableQuantity: item.maxFulfillableQuantity,
medianPrice: item.medianPrice,
totalListings: item.totalListings,
},
});
// before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls.
const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId));
const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000);
if (hasRecentSku) {
console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data'));
await sleep(100);
continue;
}
// Get product detail
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
method: 'GET',
});
if (!detailResponse.ok) {
console.error('Error fetching product details:', detailResponse.statusText);
process.exit(1);
}
const detailData = await detailResponse.json();
await db.insert(schema.cards).values({
productId: item.productId,
originalProductName: item.productName,
productName: cleanProductName(item.productName),
rarityName: item.rarityName,
productLineName: item.productLineName,
productLineUrlName: item.productLineUrlName,
productStatusId: item.productStatusId,
productTypeId: item.productTypeId,
productUrlName: item.productUrlName,
setId: item.setId,
shippingCategoryId: item.shippingCategoryId,
sealed: item.sealed,
sellerListable: item.sellerListable,
foilOnly: item.foilOnly,
attack1: item.customAttributes.attack1 || null,
attack2: item.customAttributes.attack2 || null,
attack3: item.customAttributes.attack3 || null,
attack4: item.customAttributes.attack4 || null,
cardType: item.customAttributes.cardType?.[0] || null,
cardTypeB: item.customAttributes.cardTypeB || null,
energyType: item.customAttributes.energyType?.[0] || null,
flavorText: item.customAttributes.flavorText || null,
hp: item.customAttributes.hp || 0,
number: item.customAttributes.number || '',
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null,
retreatCost: item.customAttributes.retreatCost || null,
stage: item.customAttributes.stage || null,
weakness: item.customAttributes.weakness || null,
lowestPrice: item.lowestPrice,
lowestPriceWithShipping: item.lowestPriceWithShipping,
marketPrice: item.marketPrice,
maxFulfillableQuantity: item.maxFulfillableQuantity,
medianPrice: item.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
}).onDuplicateKeyUpdate({
set: {
originalProductName: item.productName,
productName: cleanProductName(item.productName),
rarityName: item.rarityName,
productLineName: item.productLineName,
productLineUrlName: item.productLineUrlName,
productStatusId: item.productStatusId,
productTypeId: item.productTypeId,
productUrlName: item.productUrlName,
setId: item.setId,
shippingCategoryId: item.shippingCategoryId,
sealed: item.sealed,
sellerListable: item.sellerListable,
foilOnly: item.foilOnly,
attack1: item.customAttributes.attack1 || null,
attack2: item.customAttributes.attack2 || null,
attack3: item.customAttributes.attack3 || null,
attack4: item.customAttributes.attack4 || null,
cardType: item.customAttributes.cardType?.[0] || null,
cardTypeB: item.customAttributes.cardTypeB || null,
energyType: item.customAttributes.energyType?.[0] || null,
flavorText: item.customAttributes.flavorText || null,
hp: item.customAttributes.hp || 0,
number: item.customAttributes.number || '',
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null,
retreatCost: item.customAttributes.retreatCost || null,
stage: item.customAttributes.stage || null,
weakness: item.customAttributes.weakness || null,
lowestPrice: item.lowestPrice,
lowestPriceWithShipping: item.lowestPriceWithShipping,
marketPrice: item.marketPrice,
maxFulfillableQuantity: item.maxFulfillableQuantity,
medianPrice: item.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
},
});
// set is...
await db.insert(schema.sets).values({
setId: detailData.setId,
setCode: detailData.setCode,
@@ -223,33 +220,7 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
});
// skus are...
const skuArray = detailData.skus.map((sku: any) => sku.sku);
//console.log(detailData.skus);
//console.log(skuArray);
// get pricing for skus
const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ skuIds: skuArray }),
});
if (!skuResponse.ok) {
console.error('Error fetching SKU pricing:', skuResponse.statusText);
process.exit(1);
}
const skuData = await skuResponse.json();
let skuMap = new Map();
for (const skuItem of skuData) {
skuMap.set(skuItem.skuId, skuItem);
}
for (const skuItem of detailData.skus) {
const pricing = skuMap.get(skuItem.sku);
//console.log(pricing);
await db.insert(schema.skus).values({
skuId: skuItem.sku,
@@ -257,21 +228,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
condition: skuItem.condition,
language: skuItem.language,
variant: skuItem.variant,
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
highestPrice: pricing?.highestPrice || null,
lowestPrice: pricing?.lowestPrice || null,
marketPrice: pricing?.marketPrice || null,
priceCount: pricing?.priceCount || 0,
}).onDuplicateKeyUpdate({
set: {
condition: skuItem.condition,
language: skuItem.language,
variant: skuItem.variant,
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
highestPrice: pricing?.highestPrice || null,
lowestPrice: pricing?.lowestPrice || null,
marketPrice: pricing?.marketPrice || null,
priceCount: pricing?.priceCount || 0,
},
});
}
@@ -284,7 +245,8 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
const buffer = await imageResponse.arrayBuffer();
await fs.writeFile(imagePath, Buffer.from(buffer));
} else {
console.error('Error fetching product image:', imageResponse.statusText);
console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
}
}
@@ -293,12 +255,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
}
await poolConnection.end();
start += size;
}
}
// clear the log file
await fs.rm('missing_images.log', { force: true });
syncTcgplayer();
await syncTcgplayer();
await poolConnection.end();

2
src/components/Card.astro
View File

@@ -10,7 +10,7 @@ import EnergyIcon from './EnergyIcon.astro';
const { query } = Astro.props;
const searchResults = await client.collections('cards').documents().search({
q: query,
query_by: 'productLineName,productName,setName,number,rarityName',
query_by: 'productLineName,productName,setName,number,rarityName,Artist',
per_page: 250,
});
const productIds = searchResults.hits?.map((hit: any) => hit.document.productId) ?? [];

2
src/db/schema.ts
View File

@@ -2,6 +2,7 @@ import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "dri
export const cards = mysqlTable("cards", {
productId: int().primaryKey(),
originalProductName: varchar({ length: 255 }).default("").notNull(),
productName: varchar({ length: 255 }).notNull(),
productLineName: varchar({ length: 255 }).default("").notNull(),
productLineUrlName: varchar({ length: 255 }).default("").notNull(),
@@ -37,6 +38,7 @@ export const cards = mysqlTable("cards", {
retreatCost: varchar({ length: 100 }),
stage: varchar({ length: 100 }),
weakness: varchar({ length: 100 }),
Artist: varchar({ length: 255 }),
});
export const sets = mysqlTable("sets", {