pokemon/scripts/preload-pricehistory.ts

import chalk from 'chalk';
import { db, ClosePool } from '../src/db/index.ts';
import { sql } from 'drizzle-orm';
import { skus, priceHistory } from '../src/db/schema.ts';
import { toSnakeCase } from 'drizzle-orm/casing';

import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const productPath = path.join(__dirname, 'products.log');

const sleep = (ms: number) => {
  return new Promise(resolve => setTimeout(resolve, ms));
}

const headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
}

const GetHistory = async (productId:number) => {

    let monthData;
    let quarterData;
    let annualData;

    let retries = 10;

    while (retries > 0) {
        try {
            const monthResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=month`, { headers: headers });
            if (!monthResponse.ok) {
                throw new Error(`Error fetching month data: ${monthResponse.statusText}`);
            }
            monthData = await monthResponse.json();

            const quarterResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=quarter`, { headers: headers });
            if (!quarterResponse.ok) {
                throw new Error(`Error fetching quarter data: ${quarterResponse.statusText}`);
            }
            quarterData = await quarterResponse.json();

            const annualResponse = await fetch(`https://infinite-api.tcgplayer.com/price/history/${productId}/detailed?range=annual`, { headers: headers });
            if (!annualResponse.ok) {
                throw new Error(`Error fetching annual data: ${annualResponse.statusText}`);
            }
            annualData = await annualResponse.json();
            retries = 0;
        }
        catch (error) {
            retries--;
            const err = error as Error;
            console.error(err);
            if (err.message.startsWith('Error fetching ')) await sleep(7500);
            await sleep(2500);
        }
    }


    if (annualData.result === null) {
        console.error(chalk.red(`\tNo results found for productId: ${productId}`));
        fs.appendFile(productPath, `${productId}\n`);
        return null;
    }

    let skuCount = 0;
    let priceCount = 0;
    for (const annual of annualData.result) {
        const quarter = quarterData.result?.find((r:any) => r.skuId == annual.skuId);
        const month = monthData.result?.find((r:any) => r.skuId == annual.skuId);

        const allPrices = [
            ...annual?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [],
            ...quarter?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || [],
            ...month?.buckets?.map((r:any) => { return { skuId:Number(annual.skuId), calculatedAt:r.bucketStartDate, marketPrice:Number(r.marketPrice) }; }) || []
        ].sort((a:any,b:any) => { if(a.calculatedAt<b.calculatedAt) return -1; if(a.calculatedAt>b.calculatedAt) return 1; return 0; });;
        

        const priceUpdates = allPrices.reduce((accumulator:any[],currentItem:any) => {
            if (accumulator.length === 0 || (accumulator[accumulator.length-1].marketPrice !== currentItem.marketPrice && accumulator[accumulator.length-1].calculatedAt != currentItem.calculatedAt)) {
                accumulator.push(currentItem);
            }
            return accumulator;
        },[]);

        skuCount++;
        priceCount += priceUpdates.length;
        console.log(chalk.gray(`\tSkuId: ${annual.skuId} with ${priceUpdates.length} updates`));

        await db.insert(priceHistory).values(priceUpdates).onConflictDoUpdate({
            target: [priceHistory.skuId, priceHistory.calculatedAt ],
            set: {
                marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
            },
        }).returning();

    }

    fs.appendFile(productPath, `${productId}\n`);
    return { skuCount:skuCount, priceCount:priceCount };
}



const start = Date.now();

let productSet;
try {
    const data = await fs.readFile(productPath, 'utf8');
    const lines = data.split(/\r?\n/);
    productSet = new Set(lines.map(line => line.trim()));
} catch (err) {
    productSet = new Set();
}

// problem with this product
productSet.add('632947');
productSet.add('635161');
productSet.add('642504');
productSet.add('654346');

let count = productSet.size;
console.log(chalk.green(`${count} products already done.`));

const productIds = await db.query.tcgcards.findMany({ columns: { productId: true }});
const total = productIds.length;
for (const product of productIds) {
    const productId = product.productId;
    if (productSet.has(productId.toString().trim())) {
        // console.log(chalk.blue(`ProductId: ${productId} (.../${total})`));
    } else {
        count++;
        console.log(chalk.blue(`ProductId: ${productId} (${count}/${total})`));
        await GetHistory(productId);
        //await sleep(7000);
    }
}

await ClosePool();
const end = Date.now();
const duration = (end - start) / 1000;
console.log(chalk.green(`Price history preloaded in ${duration.toFixed(2)} seconds.`));

export {};