import 'dotenv/config'; import chalk from 'chalk'; import { db, poolConnection } from '../src/db/index.ts'; import { sql, inArray, eq } from 'drizzle-orm'; import { skus, processingSkus } from '../src/db/schema.ts'; import { client } from '../src/db/typesense.ts'; const DollarToInt = (dollar: any) => { if (dollar === null) return null; return Math.round(dollar * 100); } async function resetProcessingTable() { // Use sql.raw to execute the TRUNCATE TABLE statement await db.execute(sql.raw('TRUNCATE TABLE processingSkus;')); await db.insert(processingSkus).select(db.select({skuId: skus.skuId}).from(skus)); } async function syncPrices() { const batchSize = 1000; // const skuIndex = client.collections('skus'); await resetProcessingTable(); console.log(chalk.green('Processing table reset and populated with current SKUs.')); while (true) { const batch = await db.select().from(processingSkus).limit(batchSize); if (batch.length === 0) { console.log('All SKUs processed.'); break; } const skuIds = batch.map(item => item.skuId); console.log(`${chalk.blue('Processing SKUs:')} ${chalk.gray(skuIds.join(', '))}`); const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ skuIds: skuIds }), }); if (!skuResponse.ok) { console.error('Error fetching SKU pricing:', skuResponse.statusText); process.exit(1); } const skuData = await skuResponse.json(); if (skuData.length !== batchSize) { console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`)); } const skuUpdates = skuData.map((sku: any) => { return { skuId: sku.skuId, cardId: 0, productId: 0, condition: '', language: '', variant: '', calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null, highestPrice: sku.highestPrice, lowestPrice: sku.lowestPrice, marketPrice: sku.marketPrice, priceCount: null, }}); await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({ set: { calculatedAt: sql`values(${skus.calculatedAt})`, highestPrice: sql`values(${skus.highestPrice})`, lowestPrice: sql`values(${skus.lowestPrice})`, marketPrice: sql`values(${skus.marketPrice})`, } }); // remove skus from the 'working' processingSkus table await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); } } async function indexPrices() { const skus = await db.query.skus.findMany(); await client.collections('skus').documents().import(skus.map(sku => ({ id: sku.skuId.toString(), condition: sku.condition, highestPrice: DollarToInt(sku.highestPrice), lowestPrice: DollarToInt(sku.lowestPrice), marketPrice: DollarToInt(sku.marketPrice), })), { action: 'upsert' }); } const start = Date.now(); await syncPrices(); await indexPrices(); await poolConnection.end(); const end = Date.now(); const duration = (end - start) / 1000; console.log(chalk.green(`Price sync completed in ${duration.toFixed(2)} seconds.`)); export {};