[chore] preload all price history

scripts/preload-pricehistory.ts

@@ -4,6 +4,14 @@ import { sql } from 'drizzle-orm';
 import { skus, priceHistory } from '../src/db/schema.ts';
 import { toSnakeCase } from 'drizzle-orm/casing';
 
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const productPath = path.join(__dirname, 'products.log');
 
 const sleep = (ms: number) => {
   return new Promise(resolve => setTimeout(resolve, ms));
@@ -15,32 +23,54 @@ const headers = {
 
 const GetHistory = async (productId:number) => {
 
-    const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/283926/detailed?range=month`, { headers: headers });
-    if (!monthResponse.ok) {
-        console.error('Error fetching month data:', monthResponse.statusText);
-        process.exit(1);
-    }
-    const monthData = await monthResponse.json();
+    let monthData;
+    let quarterData;
+    let annualData;
 
-    const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers });
-    if (!quarterResponse.ok) {
-        console.error('Error fetching quarter data:', quarterResponse.statusText);
-        process.exit(1);
-    }
-    const quarterData = await quarterResponse.json();
+    let retries = 10;
 
-    const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers });
-    if (!annualResponse.ok) {
-        console.error('Error fetching annual data:', annualResponse.statusText);
-        process.exit(1);
+    while (retries > 0) {
+        try {
+            const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=month`, { headers: headers });
+            if (!monthResponse.ok) {
+                throw new Error(`Error fetching month data: ${monthResponse.statusText}`);
+            }
+            monthData = await monthResponse.json();
+
+            const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers });
+            if (!quarterResponse.ok) {
+                throw new Error(`Error fetching quarter data: ${quarterResponse.statusText}`);
+            }
+            quarterData = await quarterResponse.json();
+
+            const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers });
+            if (!annualResponse.ok) {
+                throw new Error(`Error fetching annual data: ${annualResponse.statusText}`);
+            }
+            annualData = await annualResponse.json();
+            retries = 0;
+        }
+        catch (error) {
+            retries--;
+            const err = error as Error;
+            console.error(err);
+            if (err.message.startsWith('Error fetching ')) await sleep(7500);
+            await sleep(2500);
+        }
+    }
+
+
+    if (annualData.result === null) {
+        console.error(chalk.red(`\tNo results found for productId: ${productId}`));
+        fs.appendFile(productPath, `${productId}\n`);
+        return null;
     }
-    const annualData = await annualResponse.json();
 
     let skuCount = 0;
     let priceCount = 0;
     for (const annual of annualData.result) {
-        const quarter = quarterData.result.find((r:any) => r.skuId == annual.skuId);
-        const month = monthData.result.find((r:any) => r.skuId == annual.skuId);
+        const quarter = quarterData.result?.find((r:any) => r.skuId == annual.skuId);
+        const month = monthData.result?.find((r:any) => r.skuId == annual.skuId);
 
         const allPrices = [
             ...annual?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [],
@@ -60,8 +90,6 @@ const GetHistory = async (productId:number) => {
         priceCount += priceUpdates.length;
         console.log(chalk.gray(`\tSkuId: ${annual.skuId} with ${priceUpdates.length} updates`));
 
-        // if (skuCount === 1) console.log(priceUpdates);
-
         await db.insert(priceHistory).values(priceUpdates).onConflictDoUpdate({
             target: [priceHistory.skuId, priceHistory.calculatedAt ],
             set: {
@@ -71,6 +99,7 @@ const GetHistory = async (productId:number) => {
 
     }
 
+    fs.appendFile(productPath, `${productId}\n`);
     return { skuCount:skuCount, priceCount:priceCount };
 }
 
@@ -78,15 +107,36 @@ const GetHistory = async (productId:number) => {
 
 const start = Date.now();
 
+let productSet;
+try {
+    const data = await fs.readFile(productPath, 'utf8');
+    const lines = data.split(/\r?\n/);
+    productSet = new Set(lines.map(line => line.trim()));
+} catch (err) {
+    productSet = new Set();
+}
+
+// problem with this product
+productSet.add('632947');
+productSet.add('635161');
+productSet.add('642504');
+productSet.add('654346');
+
+let count = productSet.size;
+console.log(chalk.green(`${count} products already done.`));
+
 const productIds = await db.query.tcgcards.findMany({ columns: { productId: true }});
 const total = productIds.length;
-let count = 0;
 for (const product of productIds) {
-    count++;
     const productId = product.productId;
-    console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`));
-    const results = await GetHistory(productId);
-    await sleep(500);
+    if (productSet.has(productId.toString().trim())) {
+        // console.log(chalk.blue(`ProductId: ${productId} (.../${total})`));
+    } else {
+        count++;
+        console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`));
+        await GetHistory(productId);
+        //await sleep(7000);
+    }
 }
 
 await ClosePool();