diff --git a/scripts/preload-tcgplayer.ts b/scripts/preload-tcgplayer.ts index 55e5498..22e1679 100644 --- a/scripts/preload-tcgplayer.ts +++ b/scripts/preload-tcgplayer.ts @@ -130,10 +130,10 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s for (const item of data.results[0].results) { - // Check if productId already exists and skip if it does (to avoid hitting the API too much) - if (allProductIds.has(item.productId)) { - continue; - } + // // Check if productId already exists and skip if it does (to avoid hitting the API too much) + // if (allProductIds.has(item.productId)) { + // continue; + // } console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); @@ -150,76 +150,76 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s await db.insert(schema.tcgcards).values({ productId: item.productId, - productName: item.productName, + productName: detailData.productName, //productName: cleanProductName(item.productName), rarityName: item.rarityName, - productLineName: item.productLineName, - productLineUrlName: item.productLineUrlName, - productStatusId: item.productStatusId, - productTypeId: item.productTypeId, - productUrlName: item.productUrlName, - setId: item.setId, - shippingCategoryId: item.shippingCategoryId, - sealed: item.sealed, - sellerListable: item.sellerListable, - foilOnly: item.foilOnly, + productLineName: detailData.productLineName, + productLineUrlName: detailData.productLineUrlName, + productStatusId: detailData.productStatusId, + productTypeId: detailData.productTypeId, + productUrlName: detailData.productUrlName, + setId: detailData.setId, + shippingCategoryId: detailData.shippingCategoryId, + sealed: detailData.sealed, + sellerListable: detailData.sellerListable, + foilOnly: detailData.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, - energyType: item.customAttributes.energyType?.[0] || null, - flavorText: item.customAttributes.flavorText || null, + energyType: detailData.customAttributes.energyType?.[0] || null, + flavorText: detailData.customAttributes.flavorText || null, hp: getNumberOrNull(item.customAttributes.hp), - number: item.customAttributes.number || '', - releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, + number: detailData.customAttributes.number || '', + releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, - lowestPrice: item.lowestPrice, - lowestPriceWithShipping: item.lowestPriceWithShipping, - marketPrice: item.marketPrice, - maxFulfillableQuantity: item.maxFulfillableQuantity, - medianPrice: item.medianPrice, + lowestPrice: detailData.lowestPrice, + lowestPriceWithShipping: detailData.lowestPriceWithShipping, + marketPrice: detailData.marketPrice, + maxFulfillableQuantity: detailData.maxFulfillableQuantity, + medianPrice: detailData.medianPrice, totalListings: item.totalListings, Artist: detailData.formattedAttributes.Artist || null, }).onDuplicateKeyUpdate({ set: { - productName: item.productName, + productName: detailData.productName, //productName: cleanProductName(item.productName), rarityName: item.rarityName, - productLineName: item.productLineName, - productLineUrlName: item.productLineUrlName, - productStatusId: item.productStatusId, - productTypeId: item.productTypeId, - productUrlName: item.productUrlName, - setId: item.setId, - shippingCategoryId: item.shippingCategoryId, - sealed: item.sealed, - sellerListable: item.sellerListable, - foilOnly: item.foilOnly, + productLineName: detailData.productLineName, + productLineUrlName: detailData.productLineUrlName, + productStatusId: detailData.productStatusId, + productTypeId: detailData.productTypeId, + productUrlName: detailData.productUrlName, + setId: detailData.setId, + shippingCategoryId: detailData.shippingCategoryId, + sealed: detailData.sealed, + sellerListable: detailData.sellerListable, + foilOnly: detailData.foilOnly, attack1: item.customAttributes.attack1 || null, attack2: item.customAttributes.attack2 || null, attack3: item.customAttributes.attack3 || null, attack4: item.customAttributes.attack4 || null, cardType: item.customAttributes.cardType?.[0] || null, cardTypeB: item.customAttributes.cardTypeB || null, - energyType: item.customAttributes.energyType?.[0] || null, - flavorText: item.customAttributes.flavorText || null, + energyType: detailData.customAttributes.energyType?.[0] || null, + flavorText: detailData.customAttributes.flavorText || null, hp: getNumberOrNull(item.customAttributes.hp), - number: item.customAttributes.number || '', - releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, + number: detailData.customAttributes.number || '', + releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null, resistance: item.customAttributes.resistance || null, retreatCost: item.customAttributes.retreatCost || null, stage: item.customAttributes.stage || null, weakness: item.customAttributes.weakness || null, - lowestPrice: item.lowestPrice, - lowestPriceWithShipping: item.lowestPriceWithShipping, - marketPrice: item.marketPrice, - maxFulfillableQuantity: item.maxFulfillableQuantity, - medianPrice: item.medianPrice, + lowestPrice: detailData.lowestPrice, + lowestPriceWithShipping: detailData.lowestPriceWithShipping, + marketPrice: detailData.marketPrice, + maxFulfillableQuantity: detailData.maxFulfillableQuantity, + medianPrice: detailData.medianPrice, totalListings: item.totalListings, Artist: detailData.formattedAttributes.Artist || null, }, @@ -272,7 +272,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s } // be nice to the API and not send too many requests in a short time - await sleep(100); + await sleep(300); } diff --git a/scripts/sync-prices.ts b/scripts/sync-prices.ts index 3fb17db..1b48b5a 100644 --- a/scripts/sync-prices.ts +++ b/scripts/sync-prices.ts @@ -4,8 +4,14 @@ import chalk from 'chalk'; import { db, poolConnection } from '../src/db/index.ts'; import { sql, inArray, eq } from 'drizzle-orm'; import { skus, processingSkus } from '../src/db/schema.ts'; +import { client } from '../src/db/typesense.ts'; +const DollarToInt = (dollar: any) => { + if (dollar === null) return null; + return Math.round(dollar * 100); +} + async function resetProcessingTable() { // Use sql.raw to execute the TRUNCATE TABLE statement await db.execute(sql.raw('TRUNCATE TABLE processingSkus;')); @@ -14,6 +20,7 @@ async function resetProcessingTable() { async function syncPrices() { const batchSize = 1000; + // const skuIndex = client.collections('skus'); await resetProcessingTable(); console.log(chalk.green('Processing table reset and populated with current SKUs.')); @@ -46,31 +53,53 @@ async function syncPrices() { if (skuData.length !== batchSize) { console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`)); - //process.exit(1); } - for (const sku of skuData) { - await db.update(skus) - .set({ - marketPrice: sku.marketPrice, - lowestPrice: sku.lowestPrice, - highestPrice: sku.highestPrice, - priceCount: sku.priceCount, - calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null, - }) - .where(eq(skus.skuId, sku.skuId)); - } + const skuUpdates = skuData.map((sku: any) => { return { + skuId: sku.skuId, + cardId: 0, + productId: 0, + condition: '', + language: '', + variant: '', + calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null, + highestPrice: sku.highestPrice, + lowestPrice: sku.lowestPrice, + marketPrice: sku.marketPrice, + priceCount: null, + }}); + await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({ + set: { + calculatedAt: sql`values(${skus.calculatedAt})`, + highestPrice: sql`values(${skus.highestPrice})`, + lowestPrice: sql`values(${skus.lowestPrice})`, + marketPrice: sql`values(${skus.marketPrice})`, + } + }); + // remove skus from the 'working' processingSkus table await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); } - // No need to call db.end(); Drizzle ORM manages connections. +} + +async function indexPrices() { + const skus = await db.query.skus.findMany(); + + await client.collections('skus').documents().import(skus.map(sku => ({ + id: sku.skuId.toString(), + condition: sku.condition, + highestPrice: DollarToInt(sku.highestPrice), + lowestPrice: DollarToInt(sku.lowestPrice), + marketPrice: DollarToInt(sku.marketPrice), + })), { action: 'upsert' }); } const start = Date.now(); await syncPrices(); +await indexPrices(); await poolConnection.end(); const end = Date.now(); const duration = (end - start) / 1000;