From 32df33f29e07aefd3919c78c71686820cb9e33fb Mon Sep 17 00:00:00 2001 From: Thad Miller Date: Sun, 15 Feb 2026 15:25:08 -0500 Subject: [PATCH] [feat] script to updating pricing --- .gitignore | 3 ++ package.json | 3 +- scripts/preload-tcgplayer.ts | 8 +--- scripts/sync-prices.ts | 79 ++++++++++++++++++++++++++++++++++++ src/db/index.ts | 9 ++-- src/db/relations.ts | 2 +- src/db/schema.ts | 4 ++ 7 files changed, 94 insertions(+), 14 deletions(-) create mode 100644 scripts/sync-prices.ts diff --git a/.gitignore b/.gitignore index cb7f03c..ac614fe 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ pnpm-debug.log* # imges from tcgplayer public/cards/* + +# anything test +test.* diff --git a/package.json b/package.json index 3bcbd13..9450e2e 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,7 @@ "dev": "astro dev", "build": "astro build", "preview": "astro preview", - "astro": "astro", - "add-user": "ts-node scripts/add-user.ts" + "astro": "astro" }, "dependencies": { "astro": "^5.17.1", diff --git a/scripts/preload-tcgplayer.ts b/scripts/preload-tcgplayer.ts index 88bd0eb..1210b04 100644 --- a/scripts/preload-tcgplayer.ts +++ b/scripts/preload-tcgplayer.ts @@ -11,13 +11,9 @@ import chalk from 'chalk'; async function syncTcgplayer() { - // const productLines = [ - // { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }, - // { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] } - // ]; - const productLines = [ - { name: "pokemon-japan", cardType: ["Dragon", "Colorless", "Energy"] } + { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }, + { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] } ]; for (const productLine of productLines) { diff --git a/scripts/sync-prices.ts b/scripts/sync-prices.ts new file mode 100644 index 0000000..fad6e63 --- /dev/null +++ b/scripts/sync-prices.ts @@ -0,0 +1,79 @@ +import 'dotenv/config'; + +import chalk from 'chalk'; +import { db, poolConnection } from '../src/db/index.ts'; +import { sql, inArray, eq } from 'drizzle-orm'; +import { skus, processingSkus } from '../src/db/schema.ts'; + + +async function resetProcessingTable() { + // Use sql.raw to execute the TRUNCATE TABLE statement + await db.execute(sql.raw('TRUNCATE TABLE processingSkus;')); + await db.insert(processingSkus).select(db.select({skuId: skus.skuId}).from(skus)); +} + +async function syncPrices() { + const batchSize = 1000; + + await resetProcessingTable(); + console.log(chalk.green('Processing table reset and populated with current SKUs.')); + + while (true) { + const batch = await db.select().from(processingSkus).limit(batchSize); + if (batch.length === 0) { + console.log('All SKUs processed.'); + break; + } + + const skuIds = batch.map(item => item.skuId); + console.log(`${chalk.blue('Processing SKUs:')} ${chalk.gray(skuIds.join(', '))}`); + + const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ skuIds: skuIds }), + }); + + if (!skuResponse.ok) { + console.error('Error fetching SKU pricing:', skuResponse.statusText); + process.exit(1); + } + + + const skuData = await skuResponse.json(); + + if (skuData.length !== batchSize) { + console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`)); + //process.exit(1); + } + + for (const sku of skuData) { + await db.update(skus) + .set({ + marketPrice: sku.marketPrice, + lowestPrice: sku.lowPrice, + highestPrice: sku.highPrice, + priceCount: sku.priceCount, + calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null, + }) + .where(eq(skus.skuId, sku.skuId)); + } + + await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); + + } + + // No need to call db.end(); Drizzle ORM manages connections. + +} + +const start = Date.now(); +await syncPrices(); +await poolConnection.end(); +const end = Date.now(); +const duration = (end - start) / 1000; +console.log(chalk.green(`Price sync completed in ${duration.toFixed(2)} seconds.`)); + +export {}; diff --git a/src/db/index.ts b/src/db/index.ts index cc3b8d3..b787475 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -1,11 +1,10 @@ // src/db/index.ts import 'dotenv/config'; -import { relations } from './relations'; +import { relations } from './relations.ts'; import { drizzle } from 'drizzle-orm/mysql2'; -//import mysql from 'mysql2/promise'; -//import * as schema from './schema'; +import mysql from 'mysql2/promise'; -//const poolConnection = mysql.createPool(process.env.DATABASE_URL!); +export const poolConnection = mysql.createPool(process.env.DATABASE_URL!); -export const db = drizzle(process.env.DATABASE_URL!, { relations }); +export const db = drizzle({ client: poolConnection, relations: relations}); diff --git a/src/db/relations.ts b/src/db/relations.ts index 3c1fdb9..0514596 100644 --- a/src/db/relations.ts +++ b/src/db/relations.ts @@ -1,5 +1,5 @@ import { defineRelations } from "drizzle-orm"; -import * as schema from "./schema"; +import * as schema from "./schema.ts"; export const relations = defineRelations(schema, (r) => ({ skus: { diff --git a/src/db/schema.ts b/src/db/schema.ts index a8c77a4..225c920 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -61,3 +61,7 @@ export const skus = mysqlTable("skus", { (table) => [ index("productIdIdx").on(table.productId), ]); + +export const processingSkus = mysqlTable("processingSkus", { + skuId: int().primaryKey(), +});