From 171ce294f44d32d8aa685a1e061fa2ced9ac51a7 Mon Sep 17 00:00:00 2001 From: Thad Miller Date: Thu, 19 Mar 2026 22:18:06 -0400 Subject: [PATCH] [chore] refactor common functions into helper script --- scripts/{indexing.ts => pokemon-helper.ts} | 75 +++++++++++++++++++++- scripts/preload-tcgplayer.ts | 65 ++++++++++--------- scripts/reindex.ts | 6 +- scripts/sync-prices.ts | 59 ++++++++++++++--- scripts/sync-variants.ts | 47 -------------- src/db/schema.ts | 2 +- 6 files changed, 160 insertions(+), 94 deletions(-) rename scripts/{indexing.ts => pokemon-helper.ts} (54%) delete mode 100644 scripts/sync-variants.ts diff --git a/scripts/indexing.ts b/scripts/pokemon-helper.ts similarity index 54% rename from scripts/indexing.ts rename to scripts/pokemon-helper.ts index 8b003dc..a3c05b8 100644 --- a/scripts/indexing.ts +++ b/scripts/pokemon-helper.ts @@ -1,6 +1,9 @@ import chalk from 'chalk'; import { client } from '../src/db/typesense.ts'; import type { DBInstance } from '../src/db/index.ts'; +import fs from "node:fs/promises"; +import { sql } from 'drizzle-orm' + const DollarToInt = (dollar: any) => { if (dollar === null) return null; @@ -8,6 +11,31 @@ const DollarToInt = (dollar: any) => { } + +export const Sleep = (ms: number) => { + return new Promise(resolve => setTimeout(resolve, ms)); +} + + +export const FileExists = async (path: string): Promise => { + try { + await fs.access(path); + return true; + } catch { + return false; + } +} + + +export const GetNumberOrNull = (value: any): number | null => { + const number = Number(value); // Attempt to convert the value to a number + if (Number.isNaN(number)) { + return null; // Return null if the result is NaN + } + return number; // Otherwise, return the number +} + + // Delete and recreate the 'cards' index export const createCardCollection = async () => { try { @@ -101,4 +129,49 @@ export const upsertSkuCollection = async (db:DBInstance) => { marketPrice: DollarToInt(sku.marketPrice), })), { action: 'upsert' }); console.log(chalk.green('Collection "skus" indexed successfully.')); -} \ No newline at end of file +} + + + + + +export const UpdateVariants = async (db:DBInstance) => { + const updates = await db.execute(sql`update cards as c +set + product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, + sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist +from ( + select t.product_id, b.variant, + coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name, + coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, + coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, + coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type, + coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist + from tcg_cards t + join (select distinct product_id, variant from skus) b on t.product_id = b.product_id + left join tcg_overrides o on t.product_id = o.product_id + ) a +where c.product_id = a.product_id and c.variant = a.variant and +( + c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or + c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or + c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or + c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist +) +`); + console.log(`Updated ${updates.rowCount} rows in cards table`); + + const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist) +select t.product_id, b.variant, +coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name, +coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name, +coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type, +coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist +from tcg_cards t +join (select distinct product_id, variant from skus) b on t.product_id = b.product_id +left join tcg_overrides o on t.product_id = o.product_id +where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant) +`); + console.log(`Inserted ${inserts.rowCount} rows into cards table`); + +} diff --git a/scripts/preload-tcgplayer.ts b/scripts/preload-tcgplayer.ts index 008b07c..1ddddcd 100644 --- a/scripts/preload-tcgplayer.ts +++ b/scripts/preload-tcgplayer.ts @@ -5,10 +5,11 @@ import { db, ClosePool } from '../src/db/index.ts'; import fs from "node:fs/promises"; import path from "node:path"; import chalk from 'chalk'; +import * as helper from './pokemon-helper.ts'; //import util from 'util'; -async function syncTcgplayer() { +async function syncTcgplayer(cardSets:string[] = []) { const productLines = [ "pokemon", "pokemon-japan" ]; @@ -29,36 +30,21 @@ async function syncTcgplayer() { const setNames = data.results[0].aggregations.setName; for (const setName of setNames) { - console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`)); - await syncProductLine(productLine, "setName", setName.urlValue); + let processSet = true; + if (cardSets.length > 0) { + processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase())); + } + if (processSet) { + console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`)); + await syncProductLine(productLine, "setName", setName.urlValue); + } } } - console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!')); } -function sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -async function fileExists(path: string): Promise { - try { - await fs.access(path); - return true; - } catch { - return false; - } -} - -function getNumberOrNull(value: any): number | null { - const number = Number(value); // Attempt to convert the value to a number - if (Number.isNaN(number)) { - return null; // Return null if the result is NaN - } - return number; // Otherwise, return the number -} async function syncProductLine(productLine: string, field: string, fieldValue: string) { let start = 0; @@ -123,7 +109,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s for (const item of data.results[0].results) { // Check if productId already exists and skip if it does (to avoid hitting the API too much) - if (allProductIds.has(item.productId)) { + if (allProductIds.size > 0 && allProductIds.has(item.productId)) { continue; } @@ -163,7 +149,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s cardTypeB: item.customAttributes.cardTypeB || null, energyType: detailData.customAttributes.energyType?.[0] || null, flavorText: detailData.customAttributes.flavorText || null, - hp: getNumberOrNull(item.customAttributes.hp), + hp: helper.GetNumberOrNull(item.customAttributes.hp), number: detailData.customAttributes.number || '', releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, @@ -201,7 +187,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s cardTypeB: item.customAttributes.cardTypeB || null, energyType: detailData.customAttributes.energyType?.[0] || null, flavorText: detailData.customAttributes.flavorText || null, - hp: getNumberOrNull(item.customAttributes.hp), + hp: helper.GetNumberOrNull(item.customAttributes.hp), number: detailData.customAttributes.number || '', releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, @@ -218,7 +204,9 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s }, }); - + console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`); + console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`); + console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`); // set is... await db.insert(schema.sets).values({ setId: detailData.setId, @@ -255,7 +243,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s // get image if it doesn't already exist const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`); - if (!await fileExists(imagePath)) { + if (!await helper.FileExists(imagePath)) { const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`); if (imageResponse.ok) { const buffer = await imageResponse.arrayBuffer(); @@ -267,7 +255,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s } // be nice to the API and not send too many requests in a short time - await sleep(300); + await helper.Sleep(300); } @@ -277,8 +265,21 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s // clear the log file await fs.rm('missing_images.log', { force: true }); +let allProductIds = new Set(); -const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId))); +const args = process.argv.slice(2); +if (args.length === 0) { + allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId))); + await syncTcgplayer(); +} +else { + await syncTcgplayer(args); +} + +// update the card table with new/updated variants +await helper.UpdateVariants(db); + +// index the card updates +helper.upsertCardCollection(db); -await syncTcgplayer(); await ClosePool(); diff --git a/scripts/reindex.ts b/scripts/reindex.ts index 613563b..7a93288 100644 --- a/scripts/reindex.ts +++ b/scripts/reindex.ts @@ -1,10 +1,10 @@ import chalk from 'chalk'; import { db, ClosePool } from '../src/db/index.ts'; -import * as Indexing from './indexing.ts'; +import * as Indexing from './pokemon-helper.ts'; -await Indexing.createCardCollection(); -await Indexing.createSkuCollection(); +//await Indexing.createCardCollection(); +//await Indexing.createSkuCollection(); await Indexing.upsertCardCollection(db); await Indexing.upsertSkuCollection(db); await ClosePool(); diff --git a/scripts/sync-prices.ts b/scripts/sync-prices.ts index 165f637..383bfd7 100644 --- a/scripts/sync-prices.ts +++ b/scripts/sync-prices.ts @@ -3,15 +3,11 @@ import 'dotenv/config'; import chalk from 'chalk'; import { db, ClosePool } from '../src/db/index.ts'; import { sql, inArray, eq } from 'drizzle-orm'; -import { skus, processingSkus, priceHistory } from '../src/db/schema.ts'; +import { skus, processingSkus, priceHistory, salesHistory } from '../src/db/schema.ts'; import { toSnakeCase } from 'drizzle-orm/casing'; -import * as Indexing from './indexing.ts'; +import * as helper from './pokemon-helper.ts'; -function sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - async function resetProcessingTable() { // Use sql.raw to execute the TRUNCATE TABLE statement await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;')); @@ -21,6 +17,7 @@ async function resetProcessingTable() { async function syncPrices() { const batchSize = 1000; // const skuIndex = client.collections('skus'); + const updatedCards = new Set(); await resetProcessingTable(); console.log(chalk.green('Processing table reset and populated with current SKUs.')); @@ -60,7 +57,7 @@ async function syncPrices() { // remove skus from the 'working' processingSkus table await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); // be nice to the API and not send too many requests in a short time - await sleep(200); + await helper.Sleep(200); continue; } @@ -103,21 +100,63 @@ async function syncPrices() { }); console.log(chalk.cyan(`${skuRows.length} history rows added.`)); } + for (const productId of skuRows.filter(row => row.calculatedAt != null).map(row => row.productId)) { + updatedCards.add(productId); + } } // remove skus from the 'working' processingSkus table await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); // be nice to the API and not send too many requests in a short time - await sleep(200); + await helper.Sleep(200); } + return updatedCards; } +const updateLatestSales = async (updatedCards: Set) => { + for (const productId of updatedCards.values()) { + console.log(`Getting sale history for ${productId}`) + const salesResponse = await fetch(`https://mpapi.tcgplayer.com/v2/product/${productId}/latestsales`,{ + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36' + }, + body: JSON.stringify({ conditions:[], languages:[1], limit:25, listType:"All", variants:[] }), + }); + if (!salesResponse.ok) { + console.error('Error fetching sale history:', salesResponse.statusText); + process.exit(1); + } + const salesData = await salesResponse.json(); + for (const sale of salesData.data) { + const skuData = await db.query.skus.findFirst({ where: { productId: productId, variant: sale.variant, condition: sale.condition } }); + if (skuData) { + await db.insert(salesHistory).values({ + skuId: skuData.skuId, + orderDate: new Date(sale.orderDate), + title: sale.title, + customListingId: sale.customListingId, + language: sale.language, + listingType: sale.listingType, + purchasePrice: sale.purchasePrice, + quantity: sale.quantity, + shippingPrice: sale.shippingPrice + }).onConflictDoNothing(); + } + } + await helper.Sleep(500); + } +} const start = Date.now(); -await syncPrices(); -await Indexing.upsertSkuCollection(db); +const updatedCards = await syncPrices(); +await helper.upsertSkuCollection(db); +//console.log(updatedCards); +//console.log(updatedCards.size); +//await updateLatestSales(updatedCards); await ClosePool(); const end = Date.now(); const duration = (end - start) / 1000; diff --git a/scripts/sync-variants.ts b/scripts/sync-variants.ts deleted file mode 100644 index bc1be3d..0000000 --- a/scripts/sync-variants.ts +++ /dev/null @@ -1,47 +0,0 @@ -import 'dotenv/config'; -import { db, ClosePool } from '../src/db/index.ts'; -import { sql } from 'drizzle-orm' - -async function syncVariants() { - const updates = await db.execute(sql`update cards as c -set - product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, - sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist -from ( - select t.product_id, b.variant, - coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name, - coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, - coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, - coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type, - coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist - from tcg_cards t - join (select distinct product_id, variant from skus) b on t.product_id = b.product_id - left join tcg_overrides o on t.product_id = o.product_id - ) a -where c.product_id = a.product_id and c.variant = a.variant and -( - c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or - c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or - c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or - c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist -) -`); - console.log(`Updated ${updates.rowCount} rows in cards table`); - - const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist) -select t.product_id, b.variant, -coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name, -coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name, -coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type, -coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist -from tcg_cards t -join (select distinct product_id, variant from skus) b on t.product_id = b.product_id -left join tcg_overrides o on t.product_id = o.product_id -where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant) -`); - console.log(`Inserted ${inserts.rowCount} rows into cards table`); - -} - -await syncVariants(); -await ClosePool(); diff --git a/src/db/schema.ts b/src/db/schema.ts index f947357..5c16cc7 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -97,7 +97,7 @@ export const skus = pokeSchema.table('skus', { priceCount: integer(), }, (table) => [ - index('idx_product_id_condition').on(table.productId, table.variant), + index('idx_product_id_condition').on(table.productId, table.variant, table.condition), ]); export const priceHistory = pokeSchema.table('price_history', {