import chalk from 'chalk'; import { db, ClosePool } from '../src/db/index.ts'; import { sql } from 'drizzle-orm'; import { skus, priceHistory } from '../src/db/schema.ts'; import { toSnakeCase } from 'drizzle-orm/casing'; import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from 'node:url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const productPath = path.join(__dirname, 'products.log'); const sleep = (ms: number) => { return new Promise(resolve => setTimeout(resolve, ms)); } const headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36' } const GetHistory = async (productId:number) => { let monthData; let quarterData; let annualData; let retries = 10; while (retries > 0) { try { const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=month`, { headers: headers }); if (!monthResponse.ok) { throw new Error(`Error fetching month data: ${monthResponse.statusText}`); } monthData = await monthResponse.json(); const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers }); if (!quarterResponse.ok) { throw new Error(`Error fetching quarter data: ${quarterResponse.statusText}`); } quarterData = await quarterResponse.json(); const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers }); if (!annualResponse.ok) { throw new Error(`Error fetching annual data: ${annualResponse.statusText}`); } annualData = await annualResponse.json(); retries = 0; } catch (error) { retries--; const err = error as Error; console.error(err); if (err.message.startsWith('Error fetching ')) await sleep(7500); await sleep(2500); } } if (annualData.result === null) { console.error(chalk.red(`\tNo results found for productId: ${productId}`)); fs.appendFile(productPath, `${productId}\n`); return null; } let skuCount = 0; let priceCount = 0; for (const annual of annualData.result) { const quarter = quarterData.result?.find((r:any) => r.skuId == annual.skuId); const month = monthData.result?.find((r:any) => r.skuId == annual.skuId); const allPrices = [ ...annual?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [], ...quarter?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [], ...month?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [] ].sort((a:any,b:any) => { if(a.calculatedAtb.calculatedAt) return 1; return 0; });; const priceUpdates = allPrices.reduce((accumulator:any[],currentItem:any) => { if (accumulator.length === 0 || (accumulator[accumulator.length-1].marketPrice !== currentItem.marketPrice && accumulator[accumulator.length-1].calculatedAt != currentItem.calculatedAt)) { accumulator.push(currentItem); } return accumulator; },[]); skuCount++; priceCount += priceUpdates.length; console.log(chalk.gray(`\tSkuId: ${annual.skuId} with ${priceUpdates.length} updates`)); await db.insert(priceHistory).values(priceUpdates).onConflictDoUpdate({ target: [priceHistory.skuId, priceHistory.calculatedAt ], set: { marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`), }, }).returning(); } fs.appendFile(productPath, `${productId}\n`); return { skuCount:skuCount, priceCount:priceCount }; } const start = Date.now(); let productSet; try { const data = await fs.readFile(productPath, 'utf8'); const lines = data.split(/\r?\n/); productSet = new Set(lines.map(line => line.trim())); } catch (err) { productSet = new Set(); } // problem with this product productSet.add('632947'); productSet.add('635161'); productSet.add('642504'); productSet.add('654346'); let count = productSet.size; console.log(chalk.green(`${count} products already done.`)); const productIds = await db.query.tcgcards.findMany({ columns: { productId: true }}); const total = productIds.length; for (const product of productIds) { const productId = product.productId; if (productSet.has(productId.toString().trim())) { // console.log(chalk.blue(`ProductId: ${productId} (.../${total})`)); } else { count++; console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`)); await GetHistory(productId); //await sleep(7000); } } await ClosePool(); const end = Date.now(); const duration = (end - start) / 1000; console.log(chalk.green(`Price history preloaded in ${duration.toFixed(2)} seconds.`)); export {};