diff --git a/.gitignore b/.gitignore index ac614fe..a32ceec 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ public/cards/* # anything test test.* + +# any logs +*.log diff --git a/scripts/preload-searchindex.ts b/scripts/preload-searchindex.ts index a93079d..ec32a4d 100644 --- a/scripts/preload-searchindex.ts +++ b/scripts/preload-searchindex.ts @@ -8,9 +8,9 @@ async function createCollection(client: Client) { // Delete the collection if it already exists to ensure a clean slate try { const response = await client.collections('cards').delete(); - console.log(`Collection "cards" deleted successfully:`, response); + //console.log(`Collection "cards" deleted successfully:`, response); } catch (error) { - console.error(`Error deleting collection "cards":`, error); + //console.error(`Error deleting collection "cards":`, error); } // Create the collection with the specified schema @@ -30,6 +30,7 @@ async function createCollection(client: Client) { { name: 'cardType', type: 'string', facet: true }, { name: 'energyType', type: 'string', facet: true }, { name: 'number', type: 'string' }, + { name: 'Artist', type: 'string' }, ], default_sorting_field: 'productId', }); @@ -59,6 +60,7 @@ async function preloadSearchIndex() { cardType: card.cardType || "", energyType: card.energyType || "", number: card.number, + Artist: card.Artist || "", })), { action: 'upsert' }); console.log(chalk.green('Search index preloaded with Pokémon cards.')); diff --git a/scripts/preload-tcgplayer.ts b/scripts/preload-tcgplayer.ts index 1210b04..6729539 100644 --- a/scripts/preload-tcgplayer.ts +++ b/scripts/preload-tcgplayer.ts @@ -1,10 +1,9 @@ import 'dotenv/config'; -import { drizzle } from 'drizzle-orm/mysql2'; -import mysql from 'mysql2/promise'; import * as schema from '../src/db/schema.ts'; +import { db, poolConnection } from '../src/db/index.ts'; + import fs from "node:fs/promises"; import path from "node:path"; -import { eq } from 'drizzle-orm'; import chalk from 'chalk'; //import util from 'util'; @@ -12,8 +11,12 @@ import chalk from 'chalk'; async function syncTcgplayer() { const productLines = [ - { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }, - { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] } + { name: "pokemon", rarityName: ["Common", "Uncommon", "Promo", "Rare", "Ultra Rare", "Holo Rare", "Code Card", "Secret Rare", + "Illustration Rare", "Double Rare", "Shiny Holo Rare", "Special Illustration Rare", "Classic Collection", "Shiny Rare", + "Hyper Rare", "Unconfirmed", "ACE SPEC Rare", "Prism Rare", "Radiant Rare", "Rare BREAK", "Rare Ace", "Shiny Ultra Rare", + "Amazing Rare", "Mega Attack Rare", "Mega Hyper Rare", "Black White Rare" + ] }, + { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }, ]; for (const productLine of productLines) { @@ -21,7 +24,7 @@ async function syncTcgplayer() { if (key === "name") continue; for (const value of values) { console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`); - await syncProductLineEnergyType(productLine.name, key, value); + await syncProductLine(productLine.name, key, value); } } } @@ -33,6 +36,14 @@ function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } +function cleanProductName(name: string): string { + // remove TCGPlayer crap + name = name.replace(/ - .*$/, ''); + name = name.replace(/ \[.*\]/, ''); + name = name.replace(/ \(.*\)/, ''); + return name.trim(); +} + async function fileExists(path: string): Promise { try { await fs.access(path); @@ -42,7 +53,7 @@ async function fileExists(path: string): Promise { } } -async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) { +async function syncProductLine(productLine: string, field: string, fieldValue: string) { let start = 0; let size = 50; let total = 1000000; @@ -50,13 +61,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie while (start < total) { console.log(` Fetching items ${start} to ${start + size} of ${total}...`); - - let d = { + const d = { "algorithm":"sales_dismax", "from":start, "size":size, "filters":{ - "term":{"productLineName":[productLine]}, + "term":{"productLineName":[productLine], [field]:[fieldValue]} , "range":{}, "match":{} }, @@ -83,7 +93,6 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie }, "sort":{} }; - d.filters.term[field] = [fieldValue]; //console.log(util.inspect(d, { depth: null })); //process.exit(1); @@ -104,20 +113,24 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie const data = await response.json(); total = data.results[0].totalResults; - //console.log(data); - - const poolConnection = mysql.createPool({ - uri: process.env.DATABASE_URL, - }); - - const db = drizzle(poolConnection, { schema, mode: 'default' }); - for (const item of data.results[0].results) { console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); + // Get product detail + const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { + method: 'GET', + }); + if (!detailResponse.ok) { + console.error('Error fetching product details:', detailResponse.statusText); + process.exit(1); + } + const detailData = await detailResponse.json(); + + await db.insert(schema.cards).values({ productId: item.productId, - productName: item.productName, + originalProductName: item.productName, + productName: cleanProductName(item.productName), rarityName: item.rarityName, productLineName: item.productLineName, productLineUrlName: item.productLineUrlName, @@ -150,9 +163,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie maxFulfillableQuantity: item.maxFulfillableQuantity, medianPrice: item.medianPrice, totalListings: item.totalListings, + Artist: detailData.formattedAttributes.Artist || null, }).onDuplicateKeyUpdate({ set: { - productName: item.productName, + originalProductName: item.productName, + productName: cleanProductName(item.productName), rarityName: item.rarityName, productLineName: item.productLineName, productLineUrlName: item.productLineUrlName, @@ -185,30 +200,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie maxFulfillableQuantity: item.maxFulfillableQuantity, medianPrice: item.medianPrice, totalListings: item.totalListings, + Artist: detailData.formattedAttributes.Artist || null, }, }); - // before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls. - const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId)); - const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000); - if (hasRecentSku) { - console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data')); - await sleep(100); - continue; - } - - // Get product detail - const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { - method: 'GET', - }); - - if (!detailResponse.ok) { - console.error('Error fetching product details:', detailResponse.statusText); - process.exit(1); - } - - const detailData = await detailResponse.json(); + // set is... await db.insert(schema.sets).values({ setId: detailData.setId, setCode: detailData.setCode, @@ -223,33 +220,7 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie }); // skus are... - const skuArray = detailData.skus.map((sku: any) => sku.sku); - //console.log(detailData.skus); - //console.log(skuArray); - // get pricing for skus - const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ skuIds: skuArray }), - }); - - if (!skuResponse.ok) { - console.error('Error fetching SKU pricing:', skuResponse.statusText); - process.exit(1); - } - - const skuData = await skuResponse.json(); - - let skuMap = new Map(); - for (const skuItem of skuData) { - skuMap.set(skuItem.skuId, skuItem); - } - for (const skuItem of detailData.skus) { - const pricing = skuMap.get(skuItem.sku); - //console.log(pricing); await db.insert(schema.skus).values({ skuId: skuItem.sku, @@ -257,21 +228,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, - calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null, - highestPrice: pricing?.highestPrice || null, - lowestPrice: pricing?.lowestPrice || null, - marketPrice: pricing?.marketPrice || null, - priceCount: pricing?.priceCount || 0, }).onDuplicateKeyUpdate({ set: { condition: skuItem.condition, language: skuItem.language, variant: skuItem.variant, - calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null, - highestPrice: pricing?.highestPrice || null, - lowestPrice: pricing?.lowestPrice || null, - marketPrice: pricing?.marketPrice || null, - priceCount: pricing?.priceCount || 0, }, }); } @@ -284,7 +245,8 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie const buffer = await imageResponse.arrayBuffer(); await fs.writeFile(imagePath, Buffer.from(buffer)); } else { - console.error('Error fetching product image:', imageResponse.statusText); + console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText)); + await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8'); } } @@ -293,12 +255,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie } - - await poolConnection.end(); - start += size; } } +// clear the log file +await fs.rm('missing_images.log', { force: true }); -syncTcgplayer(); +await syncTcgplayer(); +await poolConnection.end(); diff --git a/src/components/Card.astro b/src/components/Card.astro index a5d237e..fea2a59 100644 --- a/src/components/Card.astro +++ b/src/components/Card.astro @@ -10,7 +10,7 @@ import EnergyIcon from './EnergyIcon.astro'; const { query } = Astro.props; const searchResults = await client.collections('cards').documents().search({ q: query, - query_by: 'productLineName,productName,setName,number,rarityName', + query_by: 'productLineName,productName,setName,number,rarityName,Artist', per_page: 250, }); const productIds = searchResults.hits?.map((hit: any) => hit.document.productId) ?? []; diff --git a/src/db/schema.ts b/src/db/schema.ts index 225c920..2a0e0b4 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -2,6 +2,7 @@ import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "dri export const cards = mysqlTable("cards", { productId: int().primaryKey(), + originalProductName: varchar({ length: 255 }).default("").notNull(), productName: varchar({ length: 255 }).notNull(), productLineName: varchar({ length: 255 }).default("").notNull(), productLineUrlName: varchar({ length: 255 }).default("").notNull(), @@ -37,6 +38,7 @@ export const cards = mysqlTable("cards", { retreatCost: varchar({ length: 100 }), stage: varchar({ length: 100 }), weakness: varchar({ length: 100 }), + Artist: varchar({ length: 255 }), }); export const sets = mysqlTable("sets", {