[feat] read tcgcollector csv

scripts/csvprices.ts

@@ -1,13 +1,14 @@
 import 'dotenv/config';
-import { db, poolConnection } from '../src/db/index.ts';
+import { db, ClosePool } from '../src/db/index.ts';
 import chalk from 'chalk';
 import fs from "fs";
 //import path from "node:path";
 import { parse, stringify, transform } from 'csv';
+import { client } from '../src/db/typesense.ts';
 
 async function PricesFromCSV() {
 
-  const inputFilePath = 'scripts/test.csv';
+  const inputFilePath = 'scripts/test.tcgcollector.csv';
   const outputFilePath = 'scripts/output.csv';
 
   // Create read and write streams
@@ -17,26 +18,44 @@ async function PricesFromCSV() {
   // Define the transformation logic
   const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
     try {
-      // Get the ProductId from the first column
-      const productId = Number(Object.values(row)[0]);
-      console.log('Processing productId:', productId);
-      
-      // Get and write all variants
-      const variants = await db.query.cards.findMany(
-        { with: { prices: true, tcgdata: true }, 
-        where: { tcgdata: { productName: row["Product Name"] } } 
-      });
-      
-      if (variants.length === 0) {
-        console.log('No variants found for productId:', productId);
+      // Specific query bsaed on tcgcollector CSV
+      const query = String(Object.values(row)[1]);
+      const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
+      const cardNumber = String(Object.values(row)[7]);
+      console.log(`${query} ${cardNumber} : ${setname}`);
+
+      // Use Typesense to find the card because we can easily use the combined fields
+      let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
+      if (cards.hits?.length === 0) {
+        // Try without card number
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
       }
-      
-      for (const variant of variants) {
+      if (cards.hits?.length === 0) {
+        // Try without set name
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
+      }
+      if (cards.hits?.length === 0) {
+        // I give up, just output the values from the csv
+        console.log(chalk.red('  - not found'));
         const newRow = { ...row };
-        newRow.Variant = variant.variant;
-        newRow.marketPrice = variant.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
+        newRow.Variant = '';
+        newRow.marketPrice = '';
         this.push(newRow);
       }
+      else {
+        for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
+          console.log(chalk.blue(`  - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
+          const variant = await db.query.cards.findFirst({
+            with: { prices: true, tcgdata: true },
+            where: { cardId: card.cardId }
+          });
+          const newRow = { ...row };
+          newRow.Variant = variant?.variant;
+          newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
+          this.push(newRow);
+        }
+      }
+
       callback();
     } catch (error) {
       callback(error);
@@ -56,13 +75,13 @@ async function PricesFromCSV() {
 
   outputStream.on('finish', () => {
     console.log(`Successfully written to ${outputFilePath}`);
-    poolConnection.end();
+    ClosePool();
   });
 
   outputStream.on('error', (error) => {
     console.error('An error occurred in the process:', error);
-    poolConnection.end();
+    ClosePool();
   });
 }
 
-await PricesFromCSV();
+await PricesFromCSV();