pokemon/scripts/sync-prices.ts

import 'dotenv/config';

import chalk from 'chalk';
import { db, ClosePool } from '../src/db/index.ts';
import { sql, inArray, eq } from 'drizzle-orm';
import { skus, processingSkus, priceHistory, salesHistory } from '../src/db/schema.ts';
import { toSnakeCase } from 'drizzle-orm/casing';
import * as helper from './pokemon-helper.ts';


async function resetProcessingTable() {
    // Use sql.raw to execute the TRUNCATE TABLE statement
    await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;'));
    await db.insert(processingSkus).select(db.select({skuId: skus.skuId}).from(skus));
}

async function syncPrices() {
    const batchSize = 1000;
    // const skuIndex = client.collections('skus');
    const updatedCards = new Set<number>();

    await resetProcessingTable();
    console.log(chalk.green('Processing table reset and populated with current SKUs.'));

    while (true) {
        const batch = await db.select().from(processingSkus).limit(batchSize);
        if (batch.length === 0) {
            console.log('All SKUs processed.');
            break;
        }

        const skuIds = batch.map(item => item.skuId);
        console.log(`${chalk.blue('Processing SKUs:')} ${chalk.gray(skuIds.join(', '))}`);

        const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
            method: 'POST',
            headers: {
            'Content-Type': 'application/json',
            },
            body: JSON.stringify({ skuIds: skuIds }),
        });

        if (!skuResponse.ok) {
            console.error('Error fetching SKU pricing:', skuResponse.statusText);
            process.exit(1);
        }
        

        const skuData = await skuResponse.json();

        if (skuData.length !== batchSize) {
            console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`));
        }

        if (skuData.length === 0) {
            console.error(chalk.red('0 SKUs, skipping DB updates.'));
            // remove skus from the 'working' processingSkus table
            await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
            // be nice to the API and not send too many requests in a short time
            await helper.Sleep(200);
            continue;
        }

        const skuUpdates = skuData.map((sku: any) => { return {
            skuId: sku.skuId,
            cardId: 0,
            productId: 0,
            condition: '',
            language: '',
            variant: '',
            calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null,
            highestPrice: sku.highestPrice,
            lowestPrice: sku.lowestPrice,
            marketPrice: sku.marketPrice,
            priceCount: null,
        }});
        const skuRows = await db.insert(skus).values(skuUpdates).onConflictDoUpdate({
            target: skus.skuId,
            set: {
                calculatedAt: sql.raw(`excluded.${toSnakeCase(skus.calculatedAt.name)}`),
                highestPrice: sql.raw(`excluded.${toSnakeCase(skus.highestPrice.name)}`),
                lowestPrice: sql.raw(`excluded.${toSnakeCase(skus.lowestPrice.name)}`),
                marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
            },
            setWhere: sql`skus.market_price is distinct from excluded.market_price`,
        }).returning();

        if (skuRows && skuRows.length > 0) {
            const skuHistory = skuRows.filter(row => row.calculatedAt != null).map(row => { return {
                skuId: row.skuId,
                calculatedAt: new Date(row.calculatedAt?.toISOString().slice(0, 10)||0),
                marketPrice: row.marketPrice,
            }});
            if (skuHistory && skuHistory.length > 0) {
                await db.insert(priceHistory).values(skuHistory).onConflictDoUpdate({
                    target: [priceHistory.skuId,priceHistory.calculatedAt],
                    set: {
                        marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
                    }
                });
                console.log(chalk.cyan(`${skuRows.length} history rows added.`));
            }
            for (const productId of skuRows.filter(row => row.calculatedAt != null).map(row => row.productId)) {
                updatedCards.add(productId);
            }
        }

        // remove skus from the 'working' processingSkus table
        await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));

        // be nice to the API and not send too many requests in a short time
        await helper.Sleep(200);
    }

    return updatedCards;
}

const updateLatestSales = async (updatedCards: Set<number>) => {
    for (const productId of updatedCards.values()) {
        console.log(`Getting sale history for ${productId}`)
        const salesResponse = await fetch(`https://mpapi.tcgplayer.com/v2/product/${productId}/latestsales`,{
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
            },
            body: JSON.stringify({ conditions:[], languages:[1], limit:25, listType:"All", variants:[] }),
        });
        if (!salesResponse.ok) {
            console.error('Error fetching sale history:', salesResponse.statusText);
            process.exit(1);
        }
        const salesData = await salesResponse.json();
        for (const sale of salesData.data) {
            const skuData = await db.query.skus.findFirst({ where: { productId: productId, variant: sale.variant, condition: sale.condition } });
            if (skuData) {
                await db.insert(salesHistory).values({
                    skuId: skuData.skuId,
                    orderDate: new Date(sale.orderDate),
                    title: sale.title,
                    customListingId: sale.customListingId,
                    language: sale.language,
                    listingType: sale.listingType,
                    purchasePrice: sale.purchasePrice,
                    quantity: sale.quantity,
                    shippingPrice: sale.shippingPrice
                }).onConflictDoNothing();
            }
        }
        await helper.Sleep(500);
    }
}

const start = Date.now();
const updatedCards = await syncPrices();
await helper.upsertSkuCollection(db);
await helper.upsertCardCollection(db);  
//console.log(updatedCards);
//console.log(updatedCards.size);
//await updateLatestSales(updatedCards);
await ClosePool();
const end = Date.now();
const duration = (end - start) / 1000;
console.log(chalk.green(`Price sync completed in ${duration.toFixed(2)} seconds.`));

export {};