[chore] refactor common functions into helper script

scripts/pokemon-helper.ts

@@ -1,6 +1,9 @@
 import chalk from 'chalk';
 import { client } from '../src/db/typesense.ts';
 import type { DBInstance } from '../src/db/index.ts';
+import fs from "node:fs/promises";
+import { sql } from 'drizzle-orm'
+
 
 const DollarToInt = (dollar: any) => {
     if (dollar === null) return null;
@@ -8,6 +11,31 @@ const DollarToInt = (dollar: any) => {
 }
 
 
+
+export const Sleep = (ms: number) => {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+
+export const FileExists = async (path: string): Promise<boolean> => {
+  try {
+    await fs.access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+
+export const GetNumberOrNull = (value: any): number | null => {
+  const number = Number(value); // Attempt to convert the value to a number
+  if (Number.isNaN(number)) {
+    return null; // Return null if the result is NaN
+  }
+  return number; // Otherwise, return the number
+}
+
+
 // Delete and recreate the 'cards' index
 export const createCardCollection = async () => {
     try {
@@ -101,4 +129,49 @@ export const upsertSkuCollection = async (db:DBInstance) => {
         marketPrice: DollarToInt(sku.marketPrice),
     })), { action: 'upsert' });
     console.log(chalk.green('Collection "skus" indexed successfully.'));
-}
+}
+
+
+
+
+
+export const UpdateVariants = async (db:DBInstance) => {
+    const updates = await db.execute(sql`update cards as c
+set 
+    product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, 
+    sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
+from (
+    select t.product_id, b.variant,
+        coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
+        coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
+        coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
+        coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
+        coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
+    from tcg_cards t
+    join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
+    left join tcg_overrides o on t.product_id = o.product_id
+    ) a
+where c.product_id = a.product_id and c.variant = a.variant and
+(
+    c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or 
+    c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or 
+    c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or 
+    c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
+)
+`);
+    console.log(`Updated ${updates.rowCount} rows in cards table`);
+
+    const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
+select t.product_id, b.variant, 
+coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
+coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name, 
+coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type, 
+coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
+from tcg_cards t
+join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
+left join tcg_overrides o on t.product_id = o.product_id
+where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
+`);
+    console.log(`Inserted ${inserts.rowCount} rows into cards table`);
+
+}

scripts/preload-tcgplayer.ts

@@ -5,10 +5,11 @@ import { db, ClosePool } from '../src/db/index.ts';
 import fs from "node:fs/promises";
 import path from "node:path";
 import chalk from 'chalk';
+import * as helper from './pokemon-helper.ts';
 //import util from 'util';
 
 
-async function syncTcgplayer() {
+async function syncTcgplayer(cardSets:string[] = []) {
 
   const productLines = [ "pokemon", "pokemon-japan" ];
 
@@ -29,36 +30,21 @@ async function syncTcgplayer() {
 
     const setNames = data.results[0].aggregations.setName;
     for (const setName of setNames) {
-      console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
+      let processSet = true;
-      await syncProductLine(productLine, "setName", setName.urlValue);
+      if (cardSets.length > 0) {
+        processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
+      }
+      if (processSet) {
+        console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
+        await syncProductLine(productLine, "setName", setName.urlValue);
+      }
     }
 
   }
 
-
   console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
 }
 
-function sleep(ms: number) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-async function fileExists(path: string): Promise<boolean> {
-  try {
-    await fs.access(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function getNumberOrNull(value: any): number | null {
-  const number = Number(value); // Attempt to convert the value to a number
-  if (Number.isNaN(number)) {
-    return null; // Return null if the result is NaN
-  }
-  return number; // Otherwise, return the number
-}
 
 async function syncProductLine(productLine: string, field: string, fieldValue: string) {
   let start = 0;
@@ -123,7 +109,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
     for (const item of data.results[0].results) {
 
       // Check if productId already exists and skip if it does (to avoid hitting the API too much)
-      if (allProductIds.has(item.productId)) {
+      if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
         continue;
       }
 
@@ -163,7 +149,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         cardTypeB: item.customAttributes.cardTypeB || null,
         energyType: detailData.customAttributes.energyType?.[0] || null,
         flavorText: detailData.customAttributes.flavorText || null,
-        hp: getNumberOrNull(item.customAttributes.hp),
+        hp: helper.GetNumberOrNull(item.customAttributes.hp),
         number: detailData.customAttributes.number || '',
         releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
         resistance: item.customAttributes.resistance || null,
@@ -201,7 +187,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
           cardTypeB: item.customAttributes.cardTypeB || null,
           energyType: detailData.customAttributes.energyType?.[0] || null,
           flavorText: detailData.customAttributes.flavorText || null,
-          hp: getNumberOrNull(item.customAttributes.hp),
+          hp: helper.GetNumberOrNull(item.customAttributes.hp),
           number: detailData.customAttributes.number || '',
           releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
           resistance: item.customAttributes.resistance || null,
@@ -218,7 +204,9 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         },
       });
 
-
+      console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
+      console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
+      console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
       // set is...
       await db.insert(schema.sets).values({
         setId: detailData.setId,
@@ -255,7 +243,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
 
       // get image if it doesn't already exist
       const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
-      if (!await fileExists(imagePath)) {
+      if (!await helper.FileExists(imagePath)) {
         const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
         if (imageResponse.ok) {
           const buffer = await imageResponse.arrayBuffer();
@@ -267,7 +255,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
       }
 
       // be nice to the API and not send too many requests in a short time
-      await sleep(300);
+      await helper.Sleep(300);
 
     }
 
@@ -277,8 +265,21 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
 
 // clear the log file
 await fs.rm('missing_images.log', { force: true });
+let allProductIds = new Set();
 
-const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
+const args = process.argv.slice(2);
+if (args.length === 0) {
+  allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
+  await syncTcgplayer();
+}
+else {
+  await syncTcgplayer(args);
+}
+
+// update the card table with new/updated variants
+await helper.UpdateVariants(db);
+
+// index the card updates
+helper.upsertCardCollection(db);
 
-await syncTcgplayer();
 await ClosePool();

scripts/reindex.ts

@@ -1,10 +1,10 @@
 import chalk from 'chalk';
 import { db, ClosePool } from '../src/db/index.ts';
-import * as Indexing from './indexing.ts';
+import * as Indexing from './pokemon-helper.ts';
 
 
-await Indexing.createCardCollection();
+//await Indexing.createCardCollection();
-await Indexing.createSkuCollection();
+//await Indexing.createSkuCollection();
 await Indexing.upsertCardCollection(db);
 await Indexing.upsertSkuCollection(db);
 await ClosePool();

scripts/sync-prices.ts

@@ -3,15 +3,11 @@ import 'dotenv/config';
 import chalk from 'chalk';
 import { db, ClosePool } from '../src/db/index.ts';
 import { sql, inArray, eq } from 'drizzle-orm';
-import { skus, processingSkus, priceHistory } from '../src/db/schema.ts';
+import { skus, processingSkus, priceHistory, salesHistory } from '../src/db/schema.ts';
 import { toSnakeCase } from 'drizzle-orm/casing';
-import * as Indexing from './indexing.ts';
+import * as helper from './pokemon-helper.ts';
 
 
-function sleep(ms: number) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
 async function resetProcessingTable() {
     // Use sql.raw to execute the TRUNCATE TABLE statement
     await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;'));
@@ -21,6 +17,7 @@ async function resetProcessingTable() {
 async function syncPrices() {
     const batchSize = 1000;
     // const skuIndex = client.collections('skus');
+    const updatedCards = new Set<number>();
 
     await resetProcessingTable();
     console.log(chalk.green('Processing table reset and populated with current SKUs.'));
@@ -60,7 +57,7 @@ async function syncPrices() {
             // remove skus from the 'working' processingSkus table
             await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
             // be nice to the API and not send too many requests in a short time
-            await sleep(200);
+            await helper.Sleep(200);
             continue;
         }
 
@@ -103,21 +100,63 @@ async function syncPrices() {
                 });
                 console.log(chalk.cyan(`${skuRows.length} history rows added.`));
             }
+            for (const productId of skuRows.filter(row => row.calculatedAt != null).map(row => row.productId)) {
+                updatedCards.add(productId);
+            }
         }
 
         // remove skus from the 'working' processingSkus table
         await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
 
         // be nice to the API and not send too many requests in a short time
-        await sleep(200);
+        await helper.Sleep(200);
     }
 
+    return updatedCards;
 }
 
+const updateLatestSales = async (updatedCards: Set<number>) => {
+    for (const productId of updatedCards.values()) {
+        console.log(`Getting sale history for ${productId}`)
+        const salesResponse = await fetch(`https://mpapi.tcgplayer.com/v2/product/${productId}/latestsales`,{
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
+            },
+            body: JSON.stringify({ conditions:[], languages:[1], limit:25, listType:"All", variants:[] }),
+        });
+        if (!salesResponse.ok) {
+            console.error('Error fetching sale history:', salesResponse.statusText);
+            process.exit(1);
+        }
+        const salesData = await salesResponse.json();
+        for (const sale of salesData.data) {
+            const skuData = await db.query.skus.findFirst({ where: { productId: productId, variant: sale.variant, condition: sale.condition } });
+            if (skuData) {
+                await db.insert(salesHistory).values({
+                    skuId: skuData.skuId,
+                    orderDate: new Date(sale.orderDate),
+                    title: sale.title,
+                    customListingId: sale.customListingId,
+                    language: sale.language,
+                    listingType: sale.listingType,
+                    purchasePrice: sale.purchasePrice,
+                    quantity: sale.quantity,
+                    shippingPrice: sale.shippingPrice
+                }).onConflictDoNothing();
+            }
+        }
+        await helper.Sleep(500);
+    }
+}
 
 const start = Date.now();
-await syncPrices();
+const updatedCards = await syncPrices();
-await Indexing.upsertSkuCollection(db);
+await helper.upsertSkuCollection(db);
+//console.log(updatedCards);
+//console.log(updatedCards.size);
+//await updateLatestSales(updatedCards);
 await ClosePool();
 const end = Date.now();
 const duration = (end - start) / 1000;

scripts/sync-variants.ts

@@ -1,47 +0,0 @@
-import 'dotenv/config';
-import { db, ClosePool } from '../src/db/index.ts';
-import { sql } from 'drizzle-orm'
-
-async function syncVariants() {
-    const updates = await db.execute(sql`update cards as c
-set 
-    product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, 
-    sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
-from (
-    select t.product_id, b.variant,
-        coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
-        coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
-        coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
-        coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
-        coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
-    from tcg_cards t
-    join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
-    left join tcg_overrides o on t.product_id = o.product_id
-    ) a
-where c.product_id = a.product_id and c.variant = a.variant and
-(
-    c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or 
-	c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or 
-    c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or 
-	c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
-)
-`);
-    console.log(`Updated ${updates.rowCount} rows in cards table`);
-
-    const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
-select t.product_id, b.variant, 
-coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
-coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name, 
-coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type, 
-coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
-from tcg_cards t
-join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
-left join tcg_overrides o on t.product_id = o.product_id
-where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
-`);
-    console.log(`Inserted ${inserts.rowCount} rows into cards table`);
-
-}
-
-await syncVariants();
-await ClosePool();

src/db/schema.ts

@@ -97,7 +97,7 @@ export const skus = pokeSchema.table('skus', {
 	priceCount: integer(),
 },
 (table) => [
-	index('idx_product_id_condition').on(table.productId, table.variant),
+	index('idx_product_id_condition').on(table.productId, table.variant, table.condition),
 ]);
 
 export const priceHistory = pokeSchema.table('price_history', {