diff --git a/scripts/preload-pricehistory.ts b/scripts/preload-pricehistory.ts new file mode 100644 index 0000000..e153f1e --- /dev/null +++ b/scripts/preload-pricehistory.ts @@ -0,0 +1,97 @@ +import chalk from 'chalk'; +import { db, ClosePool } from '../src/db/index.ts'; +import { sql } from 'drizzle-orm'; +import { skus, priceHistory } from '../src/db/schema.ts'; +import { toSnakeCase } from 'drizzle-orm/casing'; + + +const sleep = (ms: number) => { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +const headers = { + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36' +} + +const GetHistory = async (productId:number) => { + + const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/283926/detailed?range=month`, { headers: headers }); + if (!monthResponse.ok) { + console.error('Error fetching month data:', monthResponse.statusText); + process.exit(1); + } + const monthData = await monthResponse.json(); + + const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers }); + if (!quarterResponse.ok) { + console.error('Error fetching quarter data:', quarterResponse.statusText); + process.exit(1); + } + const quarterData = await quarterResponse.json(); + + const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers }); + if (!annualResponse.ok) { + console.error('Error fetching annual data:', annualResponse.statusText); + process.exit(1); + } + const annualData = await annualResponse.json(); + + let skuCount = 0; + let priceCount = 0; + for (const annual of annualData.result) { + const quarter = quarterData.result.find((r:any) => r.skuId == annual.skuId); + const month = monthData.result.find((r:any) => r.skuId == annual.skuId); + + const allPrices = [ + ...annual?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [], + ...quarter?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [], + ...month?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [] + ].sort((a:any,b:any) => { if(a.calculatedAtb.calculatedAt) return 1; return 0; });; + + + const priceUpdates = allPrices.reduce((accumulator:any[],currentItem:any) => { + if (accumulator.length === 0 || (accumulator[accumulator.length-1].marketPrice !== currentItem.marketPrice && accumulator[accumulator.length-1].calculatedAt != currentItem.calculatedAt)) { + accumulator.push(currentItem); + } + return accumulator; + },[]); + + skuCount++; + priceCount += priceUpdates.length; + console.log(chalk.gray(`\tSkuId: ${annual.skuId} with ${priceUpdates.length} updates`)); + + // if (skuCount === 1) console.log(priceUpdates); + + await db.insert(priceHistory).values(priceUpdates).onConflictDoUpdate({ + target: [priceHistory.skuId, priceHistory.calculatedAt ], + set: { + marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`), + }, + }).returning(); + + } + + return { skuCount:skuCount, priceCount:priceCount }; +} + + + +const start = Date.now(); + +const productIds = await db.query.tcgcards.findMany({ columns: { productId: true }}); +const total = productIds.length; +let count = 0; +for (const product of productIds) { + count++; + const productId = product.productId; + console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`)); + const results = await GetHistory(productId); + await sleep(500); +} + +await ClosePool(); +const end = Date.now(); +const duration = (end - start) / 1000; +console.log(chalk.green(`Price history preloaded in ${duration.toFixed(2)} seconds.`)); + +export {}; diff --git a/scripts/sync-prices.ts b/scripts/sync-prices.ts index 2d1f0b7..165f637 100644 --- a/scripts/sync-prices.ts +++ b/scripts/sync-prices.ts @@ -55,6 +55,15 @@ async function syncPrices() { console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`)); } + if (skuData.length === 0) { + console.error(chalk.red('0 SKUs, skipping DB updates.')); + // remove skus from the 'working' processingSkus table + await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); + // be nice to the API and not send too many requests in a short time + await sleep(200); + continue; + } + const skuUpdates = skuData.map((sku: any) => { return { skuId: sku.skuId, cardId: 0, @@ -75,8 +84,26 @@ async function syncPrices() { highestPrice: sql.raw(`excluded.${toSnakeCase(skus.highestPrice.name)}`), lowestPrice: sql.raw(`excluded.${toSnakeCase(skus.lowestPrice.name)}`), marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`), + }, + setWhere: sql`skus.market_price is distinct from excluded.market_price`, + }).returning(); + + if (skuRows && skuRows.length > 0) { + const skuHistory = skuRows.filter(row => row.calculatedAt != null).map(row => { return { + skuId: row.skuId, + calculatedAt: new Date(row.calculatedAt?.toISOString().slice(0, 10)||0), + marketPrice: row.marketPrice, + }}); + if (skuHistory && skuHistory.length > 0) { + await db.insert(priceHistory).values(skuHistory).onConflictDoUpdate({ + target: [priceHistory.skuId,priceHistory.calculatedAt], + set: { + marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`), + } + }); + console.log(chalk.cyan(`${skuRows.length} history rows added.`)); } - }); + } // remove skus from the 'working' processingSkus table await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); diff --git a/src/db/schema.ts b/src/db/schema.ts index f21ed23..94c1436 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,5 +1,5 @@ //import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "drizzle-orm/mysql-core" -import { integer, varchar, boolean, decimal, timestamp, index, pgSchema } from "drizzle-orm/pg-core"; +import { integer, varchar, boolean, decimal, timestamp, index, pgSchema, uniqueIndex, primaryKey } from "drizzle-orm/pg-core"; export const pokeSchema = pgSchema("pokemon"); @@ -101,12 +101,12 @@ export const skus = pokeSchema.table('skus', { ]); export const priceHistory = pokeSchema.table('price_history', { - skuId: integer().default(0).notNull(), - calculatedAt: timestamp(), + skuId: integer().notNull(), + calculatedAt: timestamp().notNull(), marketPrice: decimal({ precision: 10, scale: 2 }), }, (table) => [ - index('idx_price_history').on(table.skuId, table.calculatedAt), + primaryKey({ name: 'pk_price_history', columns: [table.skuId, table.calculatedAt] }) ]); export const processingSkus = pokeSchema.table('processing_skus', {