From c0120e3e77550e466510747c5e235705908f838a Mon Sep 17 00:00:00 2001 From: Thad Miller Date: Wed, 18 Mar 2026 13:19:58 -0400 Subject: [PATCH] [feat] read tcgcollector csv --- package-lock.json | 19 +++----------- scripts/csvprices.ts | 61 +++++++++++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7b6d982..cf53dc0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2322,6 +2322,7 @@ "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -4747,7 +4748,6 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", - "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -4902,17 +4902,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/jiti": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", - "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", - "devOptional": true, - "license": "MIT", - "peer": true, - "bin": { - "jiti": "lib/jiti-cli.mjs" - } - }, "node_modules/js-cookie": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", @@ -6248,6 +6237,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -6866,8 +6856,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/sass": { "version": "1.98.0", @@ -7308,6 +7297,7 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -7731,7 +7721,6 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", diff --git a/scripts/csvprices.ts b/scripts/csvprices.ts index 8cab35e..a50e7dd 100644 --- a/scripts/csvprices.ts +++ b/scripts/csvprices.ts @@ -1,13 +1,14 @@ import 'dotenv/config'; -import { db, poolConnection } from '../src/db/index.ts'; +import { db, ClosePool } from '../src/db/index.ts'; import chalk from 'chalk'; import fs from "fs"; //import path from "node:path"; import { parse, stringify, transform } from 'csv'; +import { client } from '../src/db/typesense.ts'; async function PricesFromCSV() { - const inputFilePath = 'scripts/test.csv'; + const inputFilePath = 'scripts/test.tcgcollector.csv'; const outputFilePath = 'scripts/output.csv'; // Create read and write streams @@ -17,26 +18,44 @@ async function PricesFromCSV() { // Define the transformation logic const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) { try { - // Get the ProductId from the first column - const productId = Number(Object.values(row)[0]); - console.log('Processing productId:', productId); - - // Get and write all variants - const variants = await db.query.cards.findMany( - { with: { prices: true, tcgdata: true }, - where: { tcgdata: { productName: row["Product Name"] } } - }); - - if (variants.length === 0) { - console.log('No variants found for productId:', productId); + // Specific query bsaed on tcgcollector CSV + const query = String(Object.values(row)[1]); + const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo'); + const cardNumber = String(Object.values(row)[7]); + console.log(`${query} ${cardNumber} : ${setname}`); + + // Use Typesense to find the card because we can easily use the combined fields + let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` }); + if (cards.hits?.length === 0) { + // Try without card number + cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` }); } - - for (const variant of variants) { + if (cards.hits?.length === 0) { + // Try without set name + cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` }); + } + if (cards.hits?.length === 0) { + // I give up, just output the values from the csv + console.log(chalk.red(' - not found')); const newRow = { ...row }; - newRow.Variant = variant.variant; - newRow.marketPrice = variant.prices.find(p => p.condition === 'Near Mint')?.marketPrice; + newRow.Variant = ''; + newRow.marketPrice = ''; this.push(newRow); } + else { + for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) { + console.log(chalk.blue(` - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`)); + const variant = await db.query.cards.findFirst({ + with: { prices: true, tcgdata: true }, + where: { cardId: card.cardId } + }); + const newRow = { ...row }; + newRow.Variant = variant?.variant; + newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice; + this.push(newRow); + } + } + callback(); } catch (error) { callback(error); @@ -56,13 +75,13 @@ async function PricesFromCSV() { outputStream.on('finish', () => { console.log(`Successfully written to ${outputFilePath}`); - poolConnection.end(); + ClosePool(); }); outputStream.on('error', (error) => { console.error('An error occurred in the process:', error); - poolConnection.end(); + ClosePool(); }); } -await PricesFromCSV(); \ No newline at end of file +await PricesFromCSV();