[feat] add Artist field, and name cleanup

scripts/preload-searchindex.ts

@@ -8,9 +8,9 @@ async function createCollection(client: Client) {
     // Delete the collection if it already exists to ensure a clean slate
     try {
         const response = await client.collections('cards').delete();
-        console.log(`Collection "cards" deleted successfully:`, response);
+        //console.log(`Collection "cards" deleted successfully:`, response);
     } catch (error) {
-        console.error(`Error deleting collection "cards":`, error);
+        //console.error(`Error deleting collection "cards":`, error);
     }
 
     // Create the collection with the specified schema
@@ -30,6 +30,7 @@ async function createCollection(client: Client) {
                     { name: 'cardType', type: 'string', facet: true },
                     { name: 'energyType', type: 'string', facet: true },
                     { name: 'number', type: 'string' },
+                    { name: 'Artist', type: 'string' },
                 ],
                 default_sorting_field: 'productId',
             });
@@ -59,6 +60,7 @@ async function preloadSearchIndex() {
         cardType: card.cardType || "",
         energyType: card.energyType || "",
         number: card.number,
+        Artist: card.Artist || "",
     })), { action: 'upsert' });
 
     console.log(chalk.green('Search index preloaded with Pokémon cards.'));

scripts/preload-tcgplayer.ts

@@ -1,10 +1,9 @@
 import 'dotenv/config';
-import { drizzle } from 'drizzle-orm/mysql2';
-import mysql from 'mysql2/promise';
 import * as schema from '../src/db/schema.ts';
+import { db, poolConnection } from '../src/db/index.ts';
+
 import fs from "node:fs/promises";
 import path from "node:path";
-import { eq } from 'drizzle-orm';
 import chalk from 'chalk';
 //import util from 'util';
 
@@ -12,8 +11,12 @@ import chalk from 'chalk';
 async function syncTcgplayer() {
 
   const productLines = [
-    { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
-    { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }
+    { name: "pokemon", rarityName: ["Common", "Uncommon", "Promo", "Rare", "Ultra Rare", "Holo Rare", "Code Card", "Secret Rare", 
+      "Illustration Rare", "Double Rare", "Shiny Holo Rare", "Special Illustration Rare", "Classic Collection", "Shiny Rare", 
+      "Hyper Rare", "Unconfirmed", "ACE SPEC Rare", "Prism Rare", "Radiant Rare", "Rare BREAK", "Rare Ace", "Shiny Ultra Rare", 
+      "Amazing Rare", "Mega Attack Rare", "Mega Hyper Rare", "Black White Rare"
+    ] },
+    { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
   ];
 
   for (const productLine of productLines) {
@@ -21,7 +24,7 @@ async function syncTcgplayer() {
       if (key === "name") continue;
       for (const value of values) {
         console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
-        await syncProductLineEnergyType(productLine.name, key, value);
+        await syncProductLine(productLine.name, key, value);
       }
     }
   }
@@ -33,6 +36,14 @@ function sleep(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+function cleanProductName(name: string): string {
+  // remove TCGPlayer crap
+  name = name.replace(/ - .*$/, '');
+  name = name.replace(/ \[.*\]/, '');
+  name = name.replace(/ \(.*\)/, '');
+  return name.trim();
+}
+
 async function fileExists(path: string): Promise<boolean> {
   try {
     await fs.access(path);
@@ -42,7 +53,7 @@ async function fileExists(path: string): Promise<boolean> {
   }
 }
 
-async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) {
+async function syncProductLine(productLine: string, field: string, fieldValue: string) {
   let start = 0;
   let size = 50;
   let total = 1000000;
@@ -50,13 +61,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
   while (start < total) {
     console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
 
-
-    let d = {
+    const d = {
           "algorithm":"sales_dismax",
           "from":start,
           "size":size,
           "filters":{
-              "term":{"productLineName":[productLine]},
+              "term":{"productLineName":[productLine], [field]:[fieldValue]} ,
               "range":{},
               "match":{}
           },
@@ -83,7 +93,6 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
           },
           "sort":{}
       };
-    d.filters.term[field] = [fieldValue];
 
     //console.log(util.inspect(d, { depth: null }));
     //process.exit(1);
@@ -104,20 +113,24 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
     const data = await response.json();
     total = data.results[0].totalResults;
 
-    //console.log(data);
-
-    const poolConnection = mysql.createPool({
-      uri: process.env.DATABASE_URL,
-    });
-
-    const db = drizzle(poolConnection, { schema, mode: 'default' });
-
     for (const item of data.results[0].results) {
       console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
 
+      // Get product detail
+      const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
+        method: 'GET',
+      });
+      if (!detailResponse.ok) {
+        console.error('Error fetching product details:', detailResponse.statusText);
+        process.exit(1);
+      }
+      const detailData = await detailResponse.json();
+
+
       await db.insert(schema.cards).values({
         productId: item.productId,
-        productName: item.productName,
+        originalProductName: item.productName,
+        productName: cleanProductName(item.productName),
         rarityName: item.rarityName,
         productLineName: item.productLineName,
         productLineUrlName: item.productLineUrlName,
@@ -150,9 +163,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
         maxFulfillableQuantity: item.maxFulfillableQuantity,
         medianPrice: item.medianPrice,
         totalListings: item.totalListings,
+        Artist: detailData.formattedAttributes.Artist || null,
       }).onDuplicateKeyUpdate({
         set: {
-          productName: item.productName,
+          originalProductName: item.productName,
+          productName: cleanProductName(item.productName),
           rarityName: item.rarityName,
           productLineName: item.productLineName,
           productLineUrlName: item.productLineUrlName,
@@ -185,30 +200,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
           maxFulfillableQuantity: item.maxFulfillableQuantity,
           medianPrice: item.medianPrice,
           totalListings: item.totalListings,
+          Artist: detailData.formattedAttributes.Artist || null,
         },
       });
 
-      // before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls.
-      const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId));
-      const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000);
-      if (hasRecentSku) {
-        console.log(chalk.blue('  Skipping details and pricing fetch since we have recent SKU data'));
-        await sleep(100);
-        continue;
-      }
-
-      // Get product detail
-      const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
-        method: 'GET',
-      });
-
-      if (!detailResponse.ok) {
-        console.error('Error fetching product details:', detailResponse.statusText);
-        process.exit(1);
-      }
-
-      const detailData = await detailResponse.json();
 
+      // set is...
       await db.insert(schema.sets).values({
         setId: detailData.setId,
         setCode: detailData.setCode,
@@ -223,33 +220,7 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
       });
 
       // skus are...
-      const skuArray = detailData.skus.map((sku: any) => sku.sku);
-      //console.log(detailData.skus);
-      //console.log(skuArray);
-      // get pricing for skus
-      const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({ skuIds: skuArray }),
-      });
-      
-      if (!skuResponse.ok) {
-        console.error('Error fetching SKU pricing:', skuResponse.statusText);
-        process.exit(1);
-      }
-
-      const skuData = await skuResponse.json();
-
-      let skuMap = new Map();
-      for (const skuItem of skuData) {
-        skuMap.set(skuItem.skuId, skuItem);
-      }
-
       for (const skuItem of detailData.skus) {
-        const pricing = skuMap.get(skuItem.sku);
-        //console.log(pricing);
 
         await db.insert(schema.skus).values({
           skuId: skuItem.sku,
@@ -257,21 +228,11 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
           condition: skuItem.condition,
           language: skuItem.language,
           variant: skuItem.variant,
-          calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
-          highestPrice: pricing?.highestPrice || null,
-          lowestPrice: pricing?.lowestPrice || null,
-          marketPrice: pricing?.marketPrice || null,
-          priceCount: pricing?.priceCount || 0,
         }).onDuplicateKeyUpdate({
           set: {
             condition: skuItem.condition,
             language: skuItem.language,
             variant: skuItem.variant,
-            calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
-            highestPrice: pricing?.highestPrice || null,
-            lowestPrice: pricing?.lowestPrice || null,
-            marketPrice: pricing?.marketPrice || null,
-            priceCount: pricing?.priceCount || 0,
           },
         });
       }
@@ -284,7 +245,8 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
           const buffer = await imageResponse.arrayBuffer();
           await fs.writeFile(imagePath, Buffer.from(buffer));
         } else {
-          console.error('Error fetching product image:', imageResponse.statusText);
+          console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
+          await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
         }
       }
 
@@ -293,12 +255,12 @@ async function syncProductLineEnergyType(productLine: string, field: string, fie
 
     }
 
-
-    await poolConnection.end();
-
     start += size;
   }
 }
 
+// clear the log file
+await fs.rm('missing_images.log', { force: true });
 
-syncTcgplayer();
+await syncTcgplayer();
+await poolConnection.end();