5 Commits

Author SHA1 Message Date
Zach Harding
04ea65eeeb hotfix for image-grow class 2026-03-18 14:53:10 -04:00
Zach Harding
9d9524e654 Merge branch 'feat/csv-prices' of papi.tkpups.com:tmiller/pokemon 2026-03-18 13:45:57 -04:00
c0120e3e77 [feat] read tcgcollector csv 2026-03-18 13:39:39 -04:00
660da7cded read/write CSV, prices from db 2026-03-18 13:26:42 -04:00
2a17654c74 [chore] sales history schema 2026-03-18 11:14:19 -04:00
6 changed files with 152 additions and 2 deletions

40
package-lock.json generated
View File

@@ -16,6 +16,7 @@
"bootstrap": "^5.3.8", "bootstrap": "^5.3.8",
"chalk": "^5.6.2", "chalk": "^5.6.2",
"chart.js": "^4.5.1", "chart.js": "^4.5.1",
"csv": "^6.4.1",
"dotenv": "^17.2.4", "dotenv": "^17.2.4",
"drizzle-orm": "^1.0.0-beta.15-859cf75", "drizzle-orm": "^1.0.0-beta.15-859cf75",
"pg": "^8.20.0", "pg": "^8.20.0",
@@ -3091,6 +3092,39 @@
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/csv": {
"version": "6.4.1",
"resolved": "https://registry.npmjs.org/csv/-/csv-6.4.1.tgz",
"integrity": "sha512-ajGosmTGnTwYyGl8STqZDu7R6LkDf3xL39XiOmliV/GufQeVUxHzTKIm4NOBCwmEuujK7B6isxs4Uqt9GcRCvA==",
"license": "MIT",
"dependencies": {
"csv-generate": "^4.5.0",
"csv-parse": "^6.1.0",
"csv-stringify": "^6.6.0",
"stream-transform": "^3.4.0"
},
"engines": {
"node": ">= 0.1.90"
}
},
"node_modules/csv-generate": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.5.0.tgz",
"integrity": "sha512-aQr/vmOKyBSBHNwYhAoXw1+kUsPnMSwmYgpNoo36rIXoG1ecWILnvPGZeQ6oUjzrWknZAD3+jfpqYOBAl4x15A==",
"license": "MIT"
},
"node_modules/csv-parse": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
"integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==",
"license": "MIT"
},
"node_modules/csv-stringify": {
"version": "6.6.0",
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz",
"integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==",
"license": "MIT"
},
"node_modules/debug": { "node_modules/debug": {
"version": "4.4.3", "version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
@@ -7093,6 +7127,12 @@
"integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/stream-transform": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.4.0.tgz",
"integrity": "sha512-QO3OGhKyeIV8p6eRQdG+W6WounFw519zk690hHCNfhgfP9bylVS+NTXsuBc7n+RsGn31UgFPGrWYIgoAbArKEw==",
"license": "MIT"
},
"node_modules/string_decoder": { "node_modules/string_decoder": {
"version": "1.3.0", "version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",

View File

@@ -17,6 +17,7 @@
"bootstrap": "^5.3.8", "bootstrap": "^5.3.8",
"chalk": "^5.6.2", "chalk": "^5.6.2",
"chart.js": "^4.5.1", "chart.js": "^4.5.1",
"csv": "^6.4.1",
"dotenv": "^17.2.4", "dotenv": "^17.2.4",
"drizzle-orm": "^1.0.0-beta.15-859cf75", "drizzle-orm": "^1.0.0-beta.15-859cf75",
"pg": "^8.20.0", "pg": "^8.20.0",

87
scripts/csvprices.ts Normal file
View File

@@ -0,0 +1,87 @@
import 'dotenv/config';
import { db, ClosePool } from '../src/db/index.ts';
import chalk from 'chalk';
import fs from "fs";
//import path from "node:path";
import { parse, stringify, transform } from 'csv';
import { client } from '../src/db/typesense.ts';
async function PricesFromCSV() {
const inputFilePath = 'scripts/test.tcgcollector.csv';
const outputFilePath = 'scripts/output.csv';
// Create read and write streams
const inputStream = fs.createReadStream(inputFilePath, 'utf8');
const outputStream = fs.createWriteStream(outputFilePath);
// Define the transformation logic
const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
try {
// Specific query bsaed on tcgcollector CSV
const query = String(Object.values(row)[1]);
const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
const cardNumber = String(Object.values(row)[7]);
console.log(`${query} ${cardNumber} : ${setname}`);
// Use Typesense to find the card because we can easily use the combined fields
let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
if (cards.hits?.length === 0) {
// Try without card number
cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
}
if (cards.hits?.length === 0) {
// Try without set name
cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
}
if (cards.hits?.length === 0) {
// I give up, just output the values from the csv
console.log(chalk.red(' - not found'));
const newRow = { ...row };
newRow.Variant = '';
newRow.marketPrice = '';
this.push(newRow);
}
else {
for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
console.log(chalk.blue(` - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
const variant = await db.query.cards.findFirst({
with: { prices: true, tcgdata: true },
where: { cardId: card.cardId }
});
const newRow = { ...row };
newRow.Variant = variant?.variant;
newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
this.push(newRow);
}
}
callback();
} catch (error) {
callback(error);
}
});
// Pipe the streams: Read -> Parse -> Transform -> Stringify -> Write
inputStream
.on('error', (error) => console.error('Input stream error:', error))
.pipe(parse({ columns: true, trim: true }))
.on('error', (error) => console.error('Parse error:', error))
.pipe(transformer)
.on('error', (error) => console.error('Transform error:', error))
.pipe(stringify({ header: true }))
.on('error', (error) => console.error('Stringify error:', error))
.pipe(outputStream);
outputStream.on('finish', () => {
console.log(`Successfully written to ${outputFilePath}`);
ClosePool();
});
outputStream.on('error', (error) => {
console.error('An error occurred in the process:', error);
ClosePool();
});
}
await PricesFromCSV();

View File

@@ -163,12 +163,12 @@ html {
.image-grow { .image-grow {
transition: box-shadow 350ms ease, transform 350ms ease; transition: box-shadow 350ms ease, transform 350ms ease;
//box-shadow: 0 2px 4px rgba(0, 0, 0, 0.24); box-shadow: 0 2px 4px rgba(0, 0, 0, 0.24);
&:hover, &:hover,
&:focus { &:focus {
box-shadow: 0 8px 10px rgba(0, 0, 0, 0.2); box-shadow: 0 8px 10px rgba(0, 0, 0, 0.2);
//transform: translateY(-0.9rem) scale(1.02); transform: translateY(-0.9rem) scale(1.02);
} }
} }

View File

@@ -8,12 +8,19 @@ export const relations = defineRelations(schema, (r) => ({
to: r.skus.skuId, to: r.skus.skuId,
}), }),
}, },
salesHistory: {
sku: r.one.skus({
from: r.salesHistory.skuId,
to: r.skus.skuId,
}),
},
skus: { skus: {
card: r.one.cards({ card: r.one.cards({
from: [r.skus.productId, r.skus.variant], from: [r.skus.productId, r.skus.variant],
to: [r.cards.productId, r.cards.variant], to: [r.cards.productId, r.cards.variant],
}), }),
history: r.many.priceHistory(), history: r.many.priceHistory(),
latestSales: r.many.salesHistory(),
}, },
cards: { cards: {
prices: r.many.skus(), prices: r.many.skus(),

View File

@@ -109,6 +109,21 @@ export const priceHistory = pokeSchema.table('price_history', {
primaryKey({ name: 'pk_price_history', columns: [table.skuId, table.calculatedAt] }) primaryKey({ name: 'pk_price_history', columns: [table.skuId, table.calculatedAt] })
]); ]);
export const salesHistory = pokeSchema.table('sales_history',{
skuId: integer().notNull(),
orderDate: timestamp().notNull(),
title: varchar({ length: 255 }),
customListingId: varchar({ length: 255 }),
language: varchar({ length: 100 }),
listingType: varchar({ length: 100 }),
purchasePrice: decimal({ precision: 10, scale: 2 }),
quantity: integer(),
shippingPrice: decimal({ precision: 10, scale: 2 })
},
(table) => [
primaryKey({ name: 'pk_sales_history', columns: [table.skuId, table.orderDate] })
]);
export const processingSkus = pokeSchema.table('processing_skus', { export const processingSkus = pokeSchema.table('processing_skus', {
skuId: integer().primaryKey(), skuId: integer().primaryKey(),
}); });