Merge branch 'master' of papi.tkpups.com:tmiller/pokemon
This commit is contained in:
zach
@@ -130,10 +130,10 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
|
||||
for (const item of data.results[0].results) {
|
||||
|
||||
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
|
||||
if (allProductIds.has(item.productId)) {
|
||||
continue;
|
||||
}
|
||||
// // Check if productId already exists and skip if it does (to avoid hitting the API too much)
|
||||
// if (allProductIds.has(item.productId)) {
|
||||
// continue;
|
||||
// }
|
||||
|
||||
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
|
||||
|
||||
@@ -150,76 +150,76 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
|
||||
await db.insert(schema.tcgcards).values({
|
||||
productId: item.productId,
|
||||
productName: item.productName,
|
||||
productName: detailData.productName,
|
||||
//productName: cleanProductName(item.productName),
|
||||
rarityName: item.rarityName,
|
||||
productLineName: item.productLineName,
|
||||
productLineUrlName: item.productLineUrlName,
|
||||
productStatusId: item.productStatusId,
|
||||
productTypeId: item.productTypeId,
|
||||
productUrlName: item.productUrlName,
|
||||
setId: item.setId,
|
||||
shippingCategoryId: item.shippingCategoryId,
|
||||
sealed: item.sealed,
|
||||
sellerListable: item.sellerListable,
|
||||
foilOnly: item.foilOnly,
|
||||
productLineName: detailData.productLineName,
|
||||
productLineUrlName: detailData.productLineUrlName,
|
||||
productStatusId: detailData.productStatusId,
|
||||
productTypeId: detailData.productTypeId,
|
||||
productUrlName: detailData.productUrlName,
|
||||
setId: detailData.setId,
|
||||
shippingCategoryId: detailData.shippingCategoryId,
|
||||
sealed: detailData.sealed,
|
||||
sellerListable: detailData.sellerListable,
|
||||
foilOnly: detailData.foilOnly,
|
||||
attack1: item.customAttributes.attack1 || null,
|
||||
attack2: item.customAttributes.attack2 || null,
|
||||
attack3: item.customAttributes.attack3 || null,
|
||||
attack4: item.customAttributes.attack4 || null,
|
||||
cardType: item.customAttributes.cardType?.[0] || null,
|
||||
cardTypeB: item.customAttributes.cardTypeB || null,
|
||||
energyType: item.customAttributes.energyType?.[0] || null,
|
||||
flavorText: item.customAttributes.flavorText || null,
|
||||
energyType: detailData.customAttributes.energyType?.[0] || null,
|
||||
flavorText: detailData.customAttributes.flavorText || null,
|
||||
hp: getNumberOrNull(item.customAttributes.hp),
|
||||
number: item.customAttributes.number || '',
|
||||
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
||||
number: detailData.customAttributes.number || '',
|
||||
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
|
||||
resistance: item.customAttributes.resistance || null,
|
||||
retreatCost: item.customAttributes.retreatCost || null,
|
||||
stage: item.customAttributes.stage || null,
|
||||
weakness: item.customAttributes.weakness || null,
|
||||
lowestPrice: item.lowestPrice,
|
||||
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
||||
marketPrice: item.marketPrice,
|
||||
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
||||
medianPrice: item.medianPrice,
|
||||
lowestPrice: detailData.lowestPrice,
|
||||
lowestPriceWithShipping: detailData.lowestPriceWithShipping,
|
||||
marketPrice: detailData.marketPrice,
|
||||
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
|
||||
medianPrice: detailData.medianPrice,
|
||||
totalListings: item.totalListings,
|
||||
Artist: detailData.formattedAttributes.Artist || null,
|
||||
}).onDuplicateKeyUpdate({
|
||||
set: {
|
||||
productName: item.productName,
|
||||
productName: detailData.productName,
|
||||
//productName: cleanProductName(item.productName),
|
||||
rarityName: item.rarityName,
|
||||
productLineName: item.productLineName,
|
||||
productLineUrlName: item.productLineUrlName,
|
||||
productStatusId: item.productStatusId,
|
||||
productTypeId: item.productTypeId,
|
||||
productUrlName: item.productUrlName,
|
||||
setId: item.setId,
|
||||
shippingCategoryId: item.shippingCategoryId,
|
||||
sealed: item.sealed,
|
||||
sellerListable: item.sellerListable,
|
||||
foilOnly: item.foilOnly,
|
||||
productLineName: detailData.productLineName,
|
||||
productLineUrlName: detailData.productLineUrlName,
|
||||
productStatusId: detailData.productStatusId,
|
||||
productTypeId: detailData.productTypeId,
|
||||
productUrlName: detailData.productUrlName,
|
||||
setId: detailData.setId,
|
||||
shippingCategoryId: detailData.shippingCategoryId,
|
||||
sealed: detailData.sealed,
|
||||
sellerListable: detailData.sellerListable,
|
||||
foilOnly: detailData.foilOnly,
|
||||
attack1: item.customAttributes.attack1 || null,
|
||||
attack2: item.customAttributes.attack2 || null,
|
||||
attack3: item.customAttributes.attack3 || null,
|
||||
attack4: item.customAttributes.attack4 || null,
|
||||
cardType: item.customAttributes.cardType?.[0] || null,
|
||||
cardTypeB: item.customAttributes.cardTypeB || null,
|
||||
energyType: item.customAttributes.energyType?.[0] || null,
|
||||
flavorText: item.customAttributes.flavorText || null,
|
||||
energyType: detailData.customAttributes.energyType?.[0] || null,
|
||||
flavorText: detailData.customAttributes.flavorText || null,
|
||||
hp: getNumberOrNull(item.customAttributes.hp),
|
||||
number: item.customAttributes.number || '',
|
||||
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
||||
number: detailData.customAttributes.number || '',
|
||||
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
|
||||
resistance: item.customAttributes.resistance || null,
|
||||
retreatCost: item.customAttributes.retreatCost || null,
|
||||
stage: item.customAttributes.stage || null,
|
||||
weakness: item.customAttributes.weakness || null,
|
||||
lowestPrice: item.lowestPrice,
|
||||
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
||||
marketPrice: item.marketPrice,
|
||||
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
||||
medianPrice: item.medianPrice,
|
||||
lowestPrice: detailData.lowestPrice,
|
||||
lowestPriceWithShipping: detailData.lowestPriceWithShipping,
|
||||
marketPrice: detailData.marketPrice,
|
||||
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
|
||||
medianPrice: detailData.medianPrice,
|
||||
totalListings: item.totalListings,
|
||||
Artist: detailData.formattedAttributes.Artist || null,
|
||||
},
|
||||
@@ -272,7 +272,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
}
|
||||
|
||||
// be nice to the API and not send too many requests in a short time
|
||||
await sleep(100);
|
||||
await sleep(300);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,14 @@ import chalk from 'chalk';
|
||||
import { db, poolConnection } from '../src/db/index.ts';
|
||||
import { sql, inArray, eq } from 'drizzle-orm';
|
||||
import { skus, processingSkus } from '../src/db/schema.ts';
|
||||
import { client } from '../src/db/typesense.ts';
|
||||
|
||||
|
||||
const DollarToInt = (dollar: any) => {
|
||||
if (dollar === null) return null;
|
||||
return Math.round(dollar * 100);
|
||||
}
|
||||
|
||||
async function resetProcessingTable() {
|
||||
// Use sql.raw to execute the TRUNCATE TABLE statement
|
||||
await db.execute(sql.raw('TRUNCATE TABLE processingSkus;'));
|
||||
@@ -14,6 +20,7 @@ async function resetProcessingTable() {
|
||||
|
||||
async function syncPrices() {
|
||||
const batchSize = 1000;
|
||||
// const skuIndex = client.collections('skus');
|
||||
|
||||
await resetProcessingTable();
|
||||
console.log(chalk.green('Processing table reset and populated with current SKUs.'));
|
||||
@@ -46,31 +53,53 @@ async function syncPrices() {
|
||||
|
||||
if (skuData.length !== batchSize) {
|
||||
console.error(chalk.yellow(`Expected ${batchSize} SKUs, got ${skuData.length}`));
|
||||
//process.exit(1);
|
||||
}
|
||||
|
||||
for (const sku of skuData) {
|
||||
await db.update(skus)
|
||||
.set({
|
||||
marketPrice: sku.marketPrice,
|
||||
lowestPrice: sku.lowestPrice,
|
||||
highestPrice: sku.highestPrice,
|
||||
priceCount: sku.priceCount,
|
||||
calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null,
|
||||
})
|
||||
.where(eq(skus.skuId, sku.skuId));
|
||||
}
|
||||
const skuUpdates = skuData.map((sku: any) => { return {
|
||||
skuId: sku.skuId,
|
||||
cardId: 0,
|
||||
productId: 0,
|
||||
condition: '',
|
||||
language: '',
|
||||
variant: '',
|
||||
calculatedAt: sku.calculatedAt ? new Date(sku.calculatedAt) : null,
|
||||
highestPrice: sku.highestPrice,
|
||||
lowestPrice: sku.lowestPrice,
|
||||
marketPrice: sku.marketPrice,
|
||||
priceCount: null,
|
||||
}});
|
||||
await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({
|
||||
set: {
|
||||
calculatedAt: sql`values(${skus.calculatedAt})`,
|
||||
highestPrice: sql`values(${skus.highestPrice})`,
|
||||
lowestPrice: sql`values(${skus.lowestPrice})`,
|
||||
marketPrice: sql`values(${skus.marketPrice})`,
|
||||
}
|
||||
});
|
||||
|
||||
// remove skus from the 'working' processingSkus table
|
||||
await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
|
||||
|
||||
}
|
||||
|
||||
// No need to call db.end(); Drizzle ORM manages connections.
|
||||
}
|
||||
|
||||
async function indexPrices() {
|
||||
const skus = await db.query.skus.findMany();
|
||||
|
||||
await client.collections('skus').documents().import(skus.map(sku => ({
|
||||
id: sku.skuId.toString(),
|
||||
condition: sku.condition,
|
||||
highestPrice: DollarToInt(sku.highestPrice),
|
||||
lowestPrice: DollarToInt(sku.lowestPrice),
|
||||
marketPrice: DollarToInt(sku.marketPrice),
|
||||
})), { action: 'upsert' });
|
||||
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
await syncPrices();
|
||||
await indexPrices();
|
||||
await poolConnection.end();
|
||||
const end = Date.now();
|
||||
const duration = (end - start) / 1000;
|
||||
|
||||
Reference in New Issue
Block a user