diff --git a/scripts/preload-pricehistory.ts b/scripts/preload-pricehistory.ts index e153f1e..a27ce44 100644 --- a/scripts/preload-pricehistory.ts +++ b/scripts/preload-pricehistory.ts @@ -4,6 +4,14 @@ import { sql } from 'drizzle-orm'; import { skus, priceHistory } from '../src/db/schema.ts'; import { toSnakeCase } from 'drizzle-orm/casing'; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const productPath = path.join(__dirname, 'products.log'); const sleep = (ms: number) => { return new Promise(resolve => setTimeout(resolve, ms)); @@ -15,32 +23,54 @@ const headers = { const GetHistory = async (productId:number) => { - const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/283926/detailed?range=month`, { headers: headers }); - if (!monthResponse.ok) { - console.error('Error fetching month data:', monthResponse.statusText); - process.exit(1); - } - const monthData = await monthResponse.json(); + let monthData; + let quarterData; + let annualData; - const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers }); - if (!quarterResponse.ok) { - console.error('Error fetching quarter data:', quarterResponse.statusText); - process.exit(1); - } - const quarterData = await quarterResponse.json(); + let retries = 10; - const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers }); - if (!annualResponse.ok) { - console.error('Error fetching annual data:', annualResponse.statusText); - process.exit(1); + while (retries > 0) { + try { + const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=month`, { headers: headers }); + if (!monthResponse.ok) { + throw new Error(`Error fetching month data: ${monthResponse.statusText}`); + } + monthData = await monthResponse.json(); + + const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers }); + if (!quarterResponse.ok) { + throw new Error(`Error fetching quarter data: ${quarterResponse.statusText}`); + } + quarterData = await quarterResponse.json(); + + const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers }); + if (!annualResponse.ok) { + throw new Error(`Error fetching annual data: ${annualResponse.statusText}`); + } + annualData = await annualResponse.json(); + retries = 0; + } + catch (error) { + retries--; + const err = error as Error; + console.error(err); + if (err.message.startsWith('Error fetching ')) await sleep(7500); + await sleep(2500); + } + } + + + if (annualData.result === null) { + console.error(chalk.red(`\tNo results found for productId: ${productId}`)); + fs.appendFile(productPath, `${productId}\n`); + return null; } - const annualData = await annualResponse.json(); let skuCount = 0; let priceCount = 0; for (const annual of annualData.result) { - const quarter = quarterData.result.find((r:any) => r.skuId == annual.skuId); - const month = monthData.result.find((r:any) => r.skuId == annual.skuId); + const quarter = quarterData.result?.find((r:any) => r.skuId == annual.skuId); + const month = monthData.result?.find((r:any) => r.skuId == annual.skuId); const allPrices = [ ...annual?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [], @@ -60,8 +90,6 @@ const GetHistory = async (productId:number) => { priceCount += priceUpdates.length; console.log(chalk.gray(`\tSkuId: ${annual.skuId} with ${priceUpdates.length} updates`)); - // if (skuCount === 1) console.log(priceUpdates); - await db.insert(priceHistory).values(priceUpdates).onConflictDoUpdate({ target: [priceHistory.skuId, priceHistory.calculatedAt ], set: { @@ -71,6 +99,7 @@ const GetHistory = async (productId:number) => { } + fs.appendFile(productPath, `${productId}\n`); return { skuCount:skuCount, priceCount:priceCount }; } @@ -78,15 +107,36 @@ const GetHistory = async (productId:number) => { const start = Date.now(); +let productSet; +try { + const data = await fs.readFile(productPath, 'utf8'); + const lines = data.split(/\r?\n/); + productSet = new Set(lines.map(line => line.trim())); +} catch (err) { + productSet = new Set(); +} + +// problem with this product +productSet.add('632947'); +productSet.add('635161'); +productSet.add('642504'); +productSet.add('654346'); + +let count = productSet.size; +console.log(chalk.green(`${count} products already done.`)); + const productIds = await db.query.tcgcards.findMany({ columns: { productId: true }}); const total = productIds.length; -let count = 0; for (const product of productIds) { - count++; const productId = product.productId; - console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`)); - const results = await GetHistory(productId); - await sleep(500); + if (productSet.has(productId.toString().trim())) { + // console.log(chalk.blue(`ProductId: ${productId} (.../${total})`)); + } else { + count++; + console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`)); + await GetHistory(productId); + //await sleep(7000); + } } await ClosePool();