[feat] read tcgcollector csv

package-lock.json

@@ -2322,6 +2322,7 @@
       "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/node": "*",
         "pg-protocol": "*",
@@ -4747,7 +4748,6 @@
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
       "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "safer-buffer": ">= 2.1.2 < 3.0.0"
       },
@@ -4902,17 +4902,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/jiti": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz",
-      "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==",
-      "devOptional": true,
-      "license": "MIT",
-      "peer": true,
-      "bin": {
-        "jiti": "lib/jiti-cli.mjs"
-      }
-    },
     "node_modules/js-cookie": {
       "version": "3.0.5",
       "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz",
@@ -6248,6 +6237,7 @@
       "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
       "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "pg-connection-string": "^2.12.0",
         "pg-pool": "^3.13.0",
@@ -6866,8 +6856,7 @@
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
-      "license": "MIT",
-      "peer": true
+      "license": "MIT"
     },
     "node_modules/sass": {
       "version": "1.98.0",
@@ -7308,6 +7297,7 @@
       "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "esbuild": "~0.27.0",
         "get-tsconfig": "^4.7.5"
@@ -7731,7 +7721,6 @@
       "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
       "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "esbuild": "^0.25.0",
         "fdir": "^6.4.4",

scripts/csvprices.ts

@@ -1,13 +1,14 @@
 import 'dotenv/config';
-import { db, poolConnection } from '../src/db/index.ts';
+import { db, ClosePool } from '../src/db/index.ts';
 import chalk from 'chalk';
 import fs from "fs";
 //import path from "node:path";
 import { parse, stringify, transform } from 'csv';
+import { client } from '../src/db/typesense.ts';
 
 async function PricesFromCSV() {
 
-  const inputFilePath = 'scripts/test.csv';
+  const inputFilePath = 'scripts/test.tcgcollector.csv';
   const outputFilePath = 'scripts/output.csv';
 
   // Create read and write streams
@@ -17,26 +18,44 @@ async function PricesFromCSV() {
   // Define the transformation logic
   const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
     try {
-      // Get the ProductId from the first column
-      const productId = Number(Object.values(row)[0]);
-      console.log('Processing productId:', productId);
-      
-      // Get and write all variants
-      const variants = await db.query.cards.findMany(
-        { with: { prices: true, tcgdata: true }, 
-        where: { tcgdata: { productName: row["Product Name"] } } 
-      });
-      
-      if (variants.length === 0) {
-        console.log('No variants found for productId:', productId);
+      // Specific query bsaed on tcgcollector CSV
+      const query = String(Object.values(row)[1]);
+      const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
+      const cardNumber = String(Object.values(row)[7]);
+      console.log(`${query} ${cardNumber} : ${setname}`);
+
+      // Use Typesense to find the card because we can easily use the combined fields
+      let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
+      if (cards.hits?.length === 0) {
+        // Try without card number
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
       }
-      
-      for (const variant of variants) {
+      if (cards.hits?.length === 0) {
+        // Try without set name
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
+      }
+      if (cards.hits?.length === 0) {
+        // I give up, just output the values from the csv
+        console.log(chalk.red('  - not found'));
         const newRow = { ...row };
-        newRow.Variant = variant.variant;
-        newRow.marketPrice = variant.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
+        newRow.Variant = '';
+        newRow.marketPrice = '';
         this.push(newRow);
       }
+      else {
+        for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
+          console.log(chalk.blue(`  - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
+          const variant = await db.query.cards.findFirst({
+            with: { prices: true, tcgdata: true },
+            where: { cardId: card.cardId }
+          });
+          const newRow = { ...row };
+          newRow.Variant = variant?.variant;
+          newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
+          this.push(newRow);
+        }
+      }
+
       callback();
     } catch (error) {
       callback(error);
@@ -56,13 +75,13 @@ async function PricesFromCSV() {
 
   outputStream.on('finish', () => {
     console.log(`Successfully written to ${outputFilePath}`);
-    poolConnection.end();
+    ClosePool();
   });
 
   outputStream.on('error', (error) => {
     console.error('An error occurred in the process:', error);
-    poolConnection.end();
+    ClosePool();
   });
 }
 
-await PricesFromCSV();
+await PricesFromCSV();