[feat] price history

scripts/preload-pricehistory.ts

@@ -0,0 +1,97 @@
+import chalk from 'chalk';
+import { db, ClosePool } from '../src/db/index.ts';
+import { sql } from 'drizzle-orm';
+import { skus, priceHistory } from '../src/db/schema.ts';
+import { toSnakeCase } from 'drizzle-orm/casing';
+
+
+const sleep = (ms: number) => {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+const headers = {
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
+}
+
+const GetHistory = async (productId:number) => {
+
+    const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/283926/detailed?range=month`, { headers: headers });
+    if (!monthResponse.ok) {
+        console.error('Error fetching month data:', monthResponse.statusText);
+        process.exit(1);
+    }
+    const monthData = await monthResponse.json();
+
+    const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers });
+    if (!quarterResponse.ok) {
+        console.error('Error fetching quarter data:', quarterResponse.statusText);
+        process.exit(1);
+    }
+    const quarterData = await quarterResponse.json();
+
+    const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers });
+    if (!annualResponse.ok) {
+        console.error('Error fetching annual data:', annualResponse.statusText);
+        process.exit(1);
+    }
+    const annualData = await annualResponse.json();
+
+    let skuCount = 0;
+    let priceCount = 0;
+    for (const annual of annualData.result) {
+        const quarter = quarterData.result.find((r:any) => r.skuId == annual.skuId);
+        const month = monthData.result.find((r:any) => r.skuId == annual.skuId);
+
+        const allPrices = [
+            ...annual?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [],
+            ...quarter?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [],
+            ...month?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || []
+        ].sort((a:any,b:any) => { if(a.calculatedAt<b.calculatedAt) return -1; if(a.calculatedAt>b.calculatedAt) return 1; return 0; });;
+        
+
+        const priceUpdates = allPrices.reduce((accumulator:any[],currentItem:any) => {
+            if (accumulator.length === 0 || (accumulator[accumulator.length-1].marketPrice !== currentItem.marketPrice && accumulator[accumulator.length-1].calculatedAt != currentItem.calculatedAt)) {
+                accumulator.push(currentItem);
+            }
+            return accumulator;
+        },[]);
+
+        skuCount++;
+        priceCount += priceUpdates.length;
+        console.log(chalk.gray(`\tSkuId: ${annual.skuId} with ${priceUpdates.length} updates`));
+
+        // if (skuCount === 1) console.log(priceUpdates);
+
+        await db.insert(priceHistory).values(priceUpdates).onConflictDoUpdate({
+            target: [priceHistory.skuId, priceHistory.calculatedAt ],
+            set: {
+                marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
+            },
+        }).returning();
+
+    }
+
+    return { skuCount:skuCount, priceCount:priceCount };
+}
+
+
+
+const start = Date.now();
+
+const productIds = await db.query.tcgcards.findMany({ columns: { productId: true }});
+const total = productIds.length;
+let count = 0;
+for (const product of productIds) {
+    count++;
+    const productId = product.productId;
+    console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`));
+    const results = await GetHistory(productId);
+    await sleep(500);
+}
+
+await ClosePool();
+const end = Date.now();
+const duration = (end - start) / 1000;
+console.log(chalk.green(`Price history preloaded in ${duration.toFixed(2)} seconds.`));
+
+export {};

scripts/sync-prices.ts

@@ -55,6 +55,15 @@ async function syncPrices() {
             console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`));
         }
 
+        if (skuData.length === 0) {
+            console.error(chalk.red('0 SKUs, skipping DB updates.'));
+            // remove skus from the 'working' processingSkus table
+            await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
+            // be nice to the API and not send too many requests in a short time
+            await sleep(200);
+            continue;
+        }
+
         const skuUpdates = skuData.map((sku: any) => { return {
             skuId: sku.skuId,
             cardId: 0,
@@ -75,8 +84,26 @@ async function syncPrices() {
                 highestPrice: sql.raw(`excluded.${toSnakeCase(skus.highestPrice.name)}`),
                 lowestPrice: sql.raw(`excluded.${toSnakeCase(skus.lowestPrice.name)}`),
                 marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
+            },
+            setWhere: sql`skus.market_price is distinct from excluded.market_price`,
+        }).returning();
+
+        if (skuRows && skuRows.length > 0) {
+            const skuHistory = skuRows.filter(row => row.calculatedAt != null).map(row => { return {
+                skuId: row.skuId,
+                calculatedAt: new Date(row.calculatedAt?.toISOString().slice(0, 10)||0),
+                marketPrice: row.marketPrice,
+            }});
+            if (skuHistory && skuHistory.length > 0) {
+                await db.insert(priceHistory).values(skuHistory).onConflictDoUpdate({
+                    target: [priceHistory.skuId,priceHistory.calculatedAt],
+                    set: {
+                        marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
+                    }
+                });
+                console.log(chalk.cyan(`${skuRows.length} history rows added.`));
             }
-        });
+        }
 
         // remove skus from the 'working' processingSkus table
         await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));

src/db/schema.ts

@@ -1,5 +1,5 @@
 //import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "drizzle-orm/mysql-core"
-import { integer, varchar, boolean, decimal, timestamp, index, pgSchema } from "drizzle-orm/pg-core";
+import { integer, varchar, boolean, decimal, timestamp, index, pgSchema, uniqueIndex, primaryKey } from "drizzle-orm/pg-core";
 
 export const pokeSchema = pgSchema("pokemon");
 
@@ -101,12 +101,12 @@ export const skus = pokeSchema.table('skus', {
 ]);
 
 export const priceHistory = pokeSchema.table('price_history', {
-	skuId: integer().default(0).notNull(),
-	calculatedAt: timestamp(),
+	skuId: integer().notNull(),
+	calculatedAt: timestamp().notNull(),
 	marketPrice: decimal({ precision: 10, scale: 2 }),
 },
 (table) => [
-	index('idx_price_history').on(table.skuId, table.calculatedAt),
+	primaryKey({ name: 'pk_price_history', columns: [table.skuId, table.calculatedAt] })
 ]);
 
 export const processingSkus = pokeSchema.table('processing_skus', {