[feat] sync now loops through card sets

scripts/preload-tcgplayer.ts

@@ -10,25 +10,32 @@ import chalk from 'chalk';
 
 async function syncTcgplayer() {
 
-  const productLines = [
+  const productLines = [ "pokemon", "pokemon-japan" ];
-    { name: "pokemon", rarityName: ["Common", "Uncommon", "Promo", "Rare", "Ultra Rare", "Holo Rare", "Code Card", "Secret Rare", 
-      "Illustration Rare", "Double Rare", "Shiny Holo Rare", "Special Illustration Rare", "Classic Collection", "Shiny Rare", 
-      "Hyper Rare", "Unconfirmed", "ACE SPEC Rare", "Prism Rare", "Radiant Rare", "Rare BREAK", "Rare Ace", "Shiny Ultra Rare", 
-      "Amazing Rare", "Mega Attack Rare", "Mega Hyper Rare", "Black White Rare"
-    ] },
-    { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
-  ];
 
+  // work from the available sets within the product line
   for (const productLine of productLines) {
-    for (const [key, values] of Object.entries(productLine)) {
+    const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
-      if (key === "name") continue;
+
-      for (const value of values) {
+    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
-        console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
+      method: 'POST',
-        await syncProductLine(productLine.name, key, value);
+      headers: {'Content-Type': 'application/json',},
-      }
+      body: JSON.stringify(d),
+    });
+    if (!response.ok) {
+      console.error('Error notifying sync completion:', response.statusText);
+      process.exit(1);
     }
+    const data = await response.json();
+
+    const setNames = data.results[0].aggregations.setName;
+    for (const setName of setNames) {
+      console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
+      await syncProductLine(productLine, "setName", setName.urlValue);
+    }
+
   }
 
+
   console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
 }
 
@@ -53,6 +60,14 @@ async function fileExists(path: string): Promise<boolean> {
   }
 }
 
+function getNumberOrNull(value: any): number | null {
+  const number = Number(value); // Attempt to convert the value to a number
+  if (Number.isNaN(number)) {
+    return null; // Return null if the result is NaN
+  }
+  return number; // Otherwise, return the number
+}
+
 async function syncProductLine(productLine: string, field: string, fieldValue: string) {
   let start = 0;
   let size = 50;
@@ -114,6 +129,12 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
     total = data.results[0].totalResults;
 
     for (const item of data.results[0].results) {
+
+      // Check if productId already exists and skip if it does (to avoid hitting the API too much)
+      if (allProductIds.has(item.productId)) {
+        continue;
+      }
+
       console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
 
       // Get product detail
@@ -150,7 +171,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         cardTypeB: item.customAttributes.cardTypeB || null,
         energyType: item.customAttributes.energyType?.[0] || null,
         flavorText: item.customAttributes.flavorText || null,
-        hp: item.customAttributes.hp || 0,
+        hp: getNumberOrNull(item.customAttributes.hp),
         number: item.customAttributes.number || '',
         releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
         resistance: item.customAttributes.resistance || null,
@@ -187,7 +208,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
           cardTypeB: item.customAttributes.cardTypeB || null,
           energyType: item.customAttributes.energyType?.[0] || null,
           flavorText: item.customAttributes.flavorText || null,
-          hp: item.customAttributes.hp || 0,
+          hp: getNumberOrNull(item.customAttributes.hp),
           number: item.customAttributes.number || '',
           releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
           resistance: item.customAttributes.resistance || null,
@@ -262,5 +283,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
 // clear the log file
 await fs.rm('missing_images.log', { force: true });
 
+const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
+
 await syncTcgplayer();
 await poolConnection.end();

src/db/schema.ts

@@ -12,17 +12,17 @@ export const cards = mysqlTable("cards", {
 	rarityName: varchar({ length: 100 }).default("").notNull(),
 	sealed: boolean().default(false).notNull(),
 	sellerListable: boolean().default(false).notNull(),
-	setId: int().default(0).notNull(),
+	setId: int(),
-	shippingCategoryId: int().default(0).notNull(),
+	shippingCategoryId: int(),
 	duplicate: boolean().default(false).notNull(),
 	foilOnly: boolean().default(false).notNull(),
-	maxFulfillableQuantity: int().default(0).notNull(),
+	maxFulfillableQuantity: int(),
-	totalListings: int().default(0).notNull(),
+	totalListings: int(),
-	score: decimal({ precision: 10, scale: 2, mode: 'number' }).default(0).notNull(),
+	score: decimal({ precision: 10, scale: 2, mode: 'number' }),
-	lowestPrice: decimal({ precision: 10, scale: 2, mode: 'number' }).default(0).notNull(),
+	lowestPrice: decimal({ precision: 10, scale: 2, mode: 'number' }),
-	lowestPriceWithShipping: decimal({ precision: 10, scale: 2, mode: 'number' }).default(0).notNull(),
+	lowestPriceWithShipping: decimal({ precision: 10, scale: 2, mode: 'number' }),
-	marketPrice: decimal({ precision: 10, scale: 2, mode: 'number' }).default(0).notNull(),
+	marketPrice: decimal({ precision: 10, scale: 2, mode: 'number' }),
-	medianPrice: decimal({ precision: 10, scale: 2, mode: 'number' }).default(0).notNull(),
+	medianPrice: decimal({ precision: 10, scale: 2, mode: 'number' }),
 	attack1: varchar({ length: 1024 }),
 	attack2: varchar({ length: 1024 }),
 	attack3: varchar({ length: 1024 }),
@@ -31,7 +31,7 @@ export const cards = mysqlTable("cards", {
 	cardTypeB: varchar({ length: 100 }),
 	energyType: varchar({ length: 100 }),
 	flavorText: varchar({ length: 1000 }),
-	hp: int().default(0).notNull(),
+	hp: int(),
 	number: varchar({ length: 50 }).default("").notNull(),
 	releaseDate: datetime(),
 	resistance: varchar({ length: 100 }),