tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[chore] price sync updates index

Browse Source
This commit is contained in:
Thad Miller
2026-03-03 07:59:44 -05:00
parent 1f521afa3a
commit 11dfcd38c1
2 changed files with 87 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
scripts/preload-tcgplayer.ts
View File

@@ -130,10 +130,10 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
for (const item of data.results[0].results) { for (const item of data.results[0].results) {
// Check if productId already exists and skip if it does (to avoid hitting the API too much) // // Check if productId already exists and skip if it does (to avoid hitting the API too much)
if (allProductIds.has(item.productId)) { // if (allProductIds.has(item.productId)) {
continue; // continue;
} // }
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
@@ -150,76 +150,76 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
await db.insert(schema.tcgcards).values({ await db.insert(schema.tcgcards).values({
productId: item.productId, productId: item.productId,
productName: item.productName, productName: detailData.productName,
//productName: cleanProductName(item.productName), //productName: cleanProductName(item.productName),
rarityName: item.rarityName, rarityName: item.rarityName,
productLineName: item.productLineName, productLineName: detailData.productLineName,
productLineUrlName: item.productLineUrlName, productLineUrlName: detailData.productLineUrlName,
productStatusId: item.productStatusId, productStatusId: detailData.productStatusId,
productTypeId: item.productTypeId, productTypeId: detailData.productTypeId,
productUrlName: item.productUrlName, productUrlName: detailData.productUrlName,
setId: item.setId, setId: detailData.setId,
shippingCategoryId: item.shippingCategoryId, shippingCategoryId: detailData.shippingCategoryId,
sealed: item.sealed, sealed: detailData.sealed,
sellerListable: item.sellerListable, sellerListable: detailData.sellerListable,
foilOnly: item.foilOnly, foilOnly: detailData.foilOnly,
attack1: item.customAttributes.attack1 || null, attack1: item.customAttributes.attack1 || null,
attack2: item.customAttributes.attack2 || null, attack2: item.customAttributes.attack2 || null,
attack3: item.customAttributes.attack3 || null, attack3: item.customAttributes.attack3 || null,
attack4: item.customAttributes.attack4 || null, attack4: item.customAttributes.attack4 || null,
cardType: item.customAttributes.cardType?.[0] || null, cardType: item.customAttributes.cardType?.[0] || null,
cardTypeB: item.customAttributes.cardTypeB || null, cardTypeB: item.customAttributes.cardTypeB || null,
energyType: item.customAttributes.energyType?.[0] || null, energyType: detailData.customAttributes.energyType?.[0] || null,
flavorText: item.customAttributes.flavorText || null, flavorText: detailData.customAttributes.flavorText || null,
hp: getNumberOrNull(item.customAttributes.hp), hp: getNumberOrNull(item.customAttributes.hp),
number: item.customAttributes.number || '', number: detailData.customAttributes.number || '',
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null, resistance: item.customAttributes.resistance || null,
retreatCost: item.customAttributes.retreatCost || null, retreatCost: item.customAttributes.retreatCost || null,
stage: item.customAttributes.stage || null, stage: item.customAttributes.stage || null,
weakness: item.customAttributes.weakness || null, weakness: item.customAttributes.weakness || null,
lowestPrice: item.lowestPrice, lowestPrice: detailData.lowestPrice,
lowestPriceWithShipping: item.lowestPriceWithShipping, lowestPriceWithShipping: detailData.lowestPriceWithShipping,
marketPrice: item.marketPrice, marketPrice: detailData.marketPrice,
maxFulfillableQuantity: item.maxFulfillableQuantity, maxFulfillableQuantity: detailData.maxFulfillableQuantity,
medianPrice: item.medianPrice, medianPrice: detailData.medianPrice,
totalListings: item.totalListings, totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null, Artist: detailData.formattedAttributes.Artist || null,
}).onDuplicateKeyUpdate({ }).onDuplicateKeyUpdate({
set: { set: {
productName: item.productName, productName: detailData.productName,
//productName: cleanProductName(item.productName), //productName: cleanProductName(item.productName),
rarityName: item.rarityName, rarityName: item.rarityName,
productLineName: item.productLineName, productLineName: detailData.productLineName,
productLineUrlName: item.productLineUrlName, productLineUrlName: detailData.productLineUrlName,
productStatusId: item.productStatusId, productStatusId: detailData.productStatusId,
productTypeId: item.productTypeId, productTypeId: detailData.productTypeId,
productUrlName: item.productUrlName, productUrlName: detailData.productUrlName,
setId: item.setId, setId: detailData.setId,
shippingCategoryId: item.shippingCategoryId, shippingCategoryId: detailData.shippingCategoryId,
sealed: item.sealed, sealed: detailData.sealed,
sellerListable: item.sellerListable, sellerListable: detailData.sellerListable,
foilOnly: item.foilOnly, foilOnly: detailData.foilOnly,
attack1: item.customAttributes.attack1 || null, attack1: item.customAttributes.attack1 || null,
attack2: item.customAttributes.attack2 || null, attack2: item.customAttributes.attack2 || null,
attack3: item.customAttributes.attack3 || null, attack3: item.customAttributes.attack3 || null,
attack4: item.customAttributes.attack4 || null, attack4: item.customAttributes.attack4 || null,
cardType: item.customAttributes.cardType?.[0] || null, cardType: item.customAttributes.cardType?.[0] || null,
cardTypeB: item.customAttributes.cardTypeB || null, cardTypeB: item.customAttributes.cardTypeB || null,
energyType: item.customAttributes.energyType?.[0] || null, energyType: detailData.customAttributes.energyType?.[0] || null,
flavorText: item.customAttributes.flavorText || null, flavorText: detailData.customAttributes.flavorText || null,
hp: getNumberOrNull(item.customAttributes.hp), hp: getNumberOrNull(item.customAttributes.hp),
number: item.customAttributes.number || '', number: detailData.customAttributes.number || '',
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null, resistance: item.customAttributes.resistance || null,
retreatCost: item.customAttributes.retreatCost || null, retreatCost: item.customAttributes.retreatCost || null,
stage: item.customAttributes.stage || null, stage: item.customAttributes.stage || null,
weakness: item.customAttributes.weakness || null, weakness: item.customAttributes.weakness || null,
lowestPrice: item.lowestPrice, lowestPrice: detailData.lowestPrice,
lowestPriceWithShipping: item.lowestPriceWithShipping, lowestPriceWithShipping: detailData.lowestPriceWithShipping,
marketPrice: item.marketPrice, marketPrice: detailData.marketPrice,
maxFulfillableQuantity: item.maxFulfillableQuantity, maxFulfillableQuantity: detailData.maxFulfillableQuantity,
medianPrice: item.medianPrice, medianPrice: detailData.medianPrice,
totalListings: item.totalListings, totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null, Artist: detailData.formattedAttributes.Artist || null,
}, },
@@ -272,7 +272,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
} }
// be nice to the API and not send too many requests in a short time // be nice to the API and not send too many requests in a short time
await sleep(100); await sleep(300);
} }

55
scripts/sync-prices.ts
View File

@@ -4,8 +4,14 @@ import chalk from 'chalk';
import { db, poolConnection } from '../src/db/index.ts'; import { db, poolConnection } from '../src/db/index.ts';
import { sql, inArray, eq } from 'drizzle-orm'; import { sql, inArray, eq } from 'drizzle-orm';
import { skus, processingSkus } from '../src/db/schema.ts'; import { skus, processingSkus } from '../src/db/schema.ts';
import { client } from '../src/db/typesense.ts';
const DollarToInt = (dollar: any) => {
if (dollar === null) return null;
return Math.round(dollar * 100);
}
async function resetProcessingTable() { async function resetProcessingTable() {
// Use sql.raw to execute the TRUNCATE TABLE statement // Use sql.raw to execute the TRUNCATE TABLE statement
await db.execute(sql.raw('TRUNCATE TABLE processingSkus;')); await db.execute(sql.raw('TRUNCATE TABLE processingSkus;'));
@@ -14,6 +20,7 @@ async function resetProcessingTable() {
async function syncPrices() { async function syncPrices() {
const batchSize = 1000; const batchSize = 1000;
// const skuIndex = client.collections('skus');
await resetProcessingTable(); await resetProcessingTable();
console.log(chalk.green('Processing table reset and populated with current SKUs.')); console.log(chalk.green('Processing table reset and populated with current SKUs.'));
@@ -46,31 +53,53 @@ async function syncPrices() {
if (skuData.length !== batchSize) { if (skuData.length !== batchSize) {
console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`)); console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`));
//process.exit(1);
} }
for (const sku of skuData) { const skuUpdates = skuData.map((sku: any) => { return {
await db.update(skus) skuId: sku.skuId,
.set({ cardId: 0,
marketPrice: sku.marketPrice, productId: 0,
lowestPrice: sku.lowestPrice, condition: '',
highestPrice: sku.highestPrice, language: '',
priceCount: sku.priceCount, variant: '',
calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null, calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null,
}) highestPrice: sku.highestPrice,
.where(eq(skus.skuId, sku.skuId)); lowestPrice: sku.lowestPrice,
} marketPrice: sku.marketPrice,
priceCount: null,
}});
await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({
set: {
calculatedAt: sql`values(${skus.calculatedAt})`,
highestPrice: sql`values(${skus.highestPrice})`,
lowestPrice: sql`values(${skus.lowestPrice})`,
marketPrice: sql`values(${skus.marketPrice})`,
}
});
// remove skus from the 'working' processingSkus table
await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds)); await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
} }
// No need to call db.end(); Drizzle ORM manages connections. }
async function indexPrices() {
const skus = await db.query.skus.findMany();
await client.collections('skus').documents().import(skus.map(sku => ({
id: sku.skuId.toString(),
condition: sku.condition,
highestPrice: DollarToInt(sku.highestPrice),
lowestPrice: DollarToInt(sku.lowestPrice),
marketPrice: DollarToInt(sku.marketPrice),
})), { action: 'upsert' });
} }
const start = Date.now(); const start = Date.now();
await syncPrices(); await syncPrices();
await indexPrices();
await poolConnection.end(); await poolConnection.end();
const end = Date.now(); const end = Date.now();
const duration = (end - start) / 1000; const duration = (end - start) / 1000;