tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[feat] read tcgcollector csv

Browse Source
This commit is contained in:
Thad Miller
2026-03-18 13:19:58 -04:00
parent 660da7cded
commit c0120e3e77
2 changed files with 44 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
package-lock.json generated
View File

@@ -2322,6 +2322,7 @@
"integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/node": "*",
"pg-protocol": "*",
@@ -4747,7 +4748,6 @@
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
"integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
"license": "MIT",
"peer": true,
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
@@ -4902,17 +4902,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/jiti": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz",
"integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==",
"devOptional": true,
"license": "MIT",
"peer": true,
"bin": {
"jiti": "lib/jiti-cli.mjs"
}
},
"node_modules/js-cookie": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz",
@@ -6248,6 +6237,7 @@
"resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
"integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==",
"license": "MIT",
"peer": true,
"dependencies": {
"pg-connection-string": "^2.12.0",
"pg-pool": "^3.13.0",
@@ -6866,8 +6856,7 @@
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/sass": {
"version": "1.98.0",
@@ -7308,6 +7297,7 @@
"integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"esbuild": "~0.27.0",
"get-tsconfig": "^4.7.5"
@@ -7731,7 +7721,6 @@
"resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
"integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
"license": "MIT",
"peer": true,
"dependencies": {
"esbuild": "^0.25.0",
"fdir": "^6.4.4",

57
scripts/csvprices.ts
View File

@@ -1,13 +1,14 @@
import 'dotenv/config';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import chalk from 'chalk';
import fs from "fs";
//import path from "node:path";
import { parse, stringify, transform } from 'csv';
import { client } from '../src/db/typesense.ts';
async function PricesFromCSV() {
const inputFilePath = 'scripts/test.csv';
const inputFilePath = 'scripts/test.tcgcollector.csv';
const outputFilePath = 'scripts/output.csv';
// Create read and write streams
@@ -17,26 +18,44 @@ async function PricesFromCSV() {
// Define the transformation logic
const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
try {
// Get the ProductId from the first column
const productId = Number(Object.values(row)[0]);
console.log('Processing productId:', productId);
// Specific query bsaed on tcgcollector CSV
const query = String(Object.values(row)[1]);
const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
const cardNumber = String(Object.values(row)[7]);
console.log(`${query} ${cardNumber} : ${setname}`);
// Get and write all variants
const variants = await db.query.cards.findMany(
{ with: { prices: true, tcgdata: true },
where: { tcgdata: { productName: row["Product Name"] } }
});
if (variants.length === 0) {
console.log('No variants found for productId:', productId);
// Use Typesense to find the card because we can easily use the combined fields
let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
if (cards.hits?.length === 0) {
// Try without card number
cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
}
for (const variant of variants) {
if (cards.hits?.length === 0) {
// Try without set name
cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
}
if (cards.hits?.length === 0) {
// I give up, just output the values from the csv
console.log(chalk.red(' - not found'));
const newRow = { ...row };
newRow.Variant = variant.variant;
newRow.marketPrice = variant.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
newRow.Variant = '';
newRow.marketPrice = '';
this.push(newRow);
}
else {
for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
console.log(chalk.blue(` - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
const variant = await db.query.cards.findFirst({
with: { prices: true, tcgdata: true },
where: { cardId: card.cardId }
});
const newRow = { ...row };
newRow.Variant = variant?.variant;
newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
this.push(newRow);
}
}
callback();
} catch (error) {
callback(error);
@@ -56,12 +75,12 @@ async function PricesFromCSV() {
outputStream.on('finish', () => {
console.log(`Successfully written to ${outputFilePath}`);
poolConnection.end();
ClosePool();
});
outputStream.on('error', (error) => {
console.error('An error occurred in the process:', error);
poolConnection.end();
ClosePool();
});
}