Compare commits
5 Commits
bc99be51ea
...
04ea65eeeb
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
04ea65eeeb | ||
|
|
9d9524e654 | ||
| c0120e3e77 | |||
| 660da7cded | |||
| 2a17654c74 |
40
package-lock.json
generated
40
package-lock.json
generated
@@ -16,6 +16,7 @@
|
||||
"bootstrap": "^5.3.8",
|
||||
"chalk": "^5.6.2",
|
||||
"chart.js": "^4.5.1",
|
||||
"csv": "^6.4.1",
|
||||
"dotenv": "^17.2.4",
|
||||
"drizzle-orm": "^1.0.0-beta.15-859cf75",
|
||||
"pg": "^8.20.0",
|
||||
@@ -3091,6 +3092,39 @@
|
||||
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/csv": {
|
||||
"version": "6.4.1",
|
||||
"resolved": "https://registry.npmjs.org/csv/-/csv-6.4.1.tgz",
|
||||
"integrity": "sha512-ajGosmTGnTwYyGl8STqZDu7R6LkDf3xL39XiOmliV/GufQeVUxHzTKIm4NOBCwmEuujK7B6isxs4Uqt9GcRCvA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"csv-generate": "^4.5.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"csv-stringify": "^6.6.0",
|
||||
"stream-transform": "^3.4.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.1.90"
|
||||
}
|
||||
},
|
||||
"node_modules/csv-generate": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.5.0.tgz",
|
||||
"integrity": "sha512-aQr/vmOKyBSBHNwYhAoXw1+kUsPnMSwmYgpNoo36rIXoG1ecWILnvPGZeQ6oUjzrWknZAD3+jfpqYOBAl4x15A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
|
||||
"integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/csv-stringify": {
|
||||
"version": "6.6.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz",
|
||||
"integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
@@ -7093,6 +7127,12 @@
|
||||
"integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/stream-transform": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.4.0.tgz",
|
||||
"integrity": "sha512-QO3OGhKyeIV8p6eRQdG+W6WounFw519zk690hHCNfhgfP9bylVS+NTXsuBc7n+RsGn31UgFPGrWYIgoAbArKEw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
"bootstrap": "^5.3.8",
|
||||
"chalk": "^5.6.2",
|
||||
"chart.js": "^4.5.1",
|
||||
"csv": "^6.4.1",
|
||||
"dotenv": "^17.2.4",
|
||||
"drizzle-orm": "^1.0.0-beta.15-859cf75",
|
||||
"pg": "^8.20.0",
|
||||
|
||||
87
scripts/csvprices.ts
Normal file
87
scripts/csvprices.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
import 'dotenv/config';
|
||||
import { db, ClosePool } from '../src/db/index.ts';
|
||||
import chalk from 'chalk';
|
||||
import fs from "fs";
|
||||
//import path from "node:path";
|
||||
import { parse, stringify, transform } from 'csv';
|
||||
import { client } from '../src/db/typesense.ts';
|
||||
|
||||
async function PricesFromCSV() {
|
||||
|
||||
const inputFilePath = 'scripts/test.tcgcollector.csv';
|
||||
const outputFilePath = 'scripts/output.csv';
|
||||
|
||||
// Create read and write streams
|
||||
const inputStream = fs.createReadStream(inputFilePath, 'utf8');
|
||||
const outputStream = fs.createWriteStream(outputFilePath);
|
||||
|
||||
// Define the transformation logic
|
||||
const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
|
||||
try {
|
||||
// Specific query bsaed on tcgcollector CSV
|
||||
const query = String(Object.values(row)[1]);
|
||||
const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
|
||||
const cardNumber = String(Object.values(row)[7]);
|
||||
console.log(`${query} ${cardNumber} : ${setname}`);
|
||||
|
||||
// Use Typesense to find the card because we can easily use the combined fields
|
||||
let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
|
||||
if (cards.hits?.length === 0) {
|
||||
// Try without card number
|
||||
cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
|
||||
}
|
||||
if (cards.hits?.length === 0) {
|
||||
// Try without set name
|
||||
cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
|
||||
}
|
||||
if (cards.hits?.length === 0) {
|
||||
// I give up, just output the values from the csv
|
||||
console.log(chalk.red(' - not found'));
|
||||
const newRow = { ...row };
|
||||
newRow.Variant = '';
|
||||
newRow.marketPrice = '';
|
||||
this.push(newRow);
|
||||
}
|
||||
else {
|
||||
for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
|
||||
console.log(chalk.blue(` - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
|
||||
const variant = await db.query.cards.findFirst({
|
||||
with: { prices: true, tcgdata: true },
|
||||
where: { cardId: card.cardId }
|
||||
});
|
||||
const newRow = { ...row };
|
||||
newRow.Variant = variant?.variant;
|
||||
newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
|
||||
this.push(newRow);
|
||||
}
|
||||
}
|
||||
|
||||
callback();
|
||||
} catch (error) {
|
||||
callback(error);
|
||||
}
|
||||
});
|
||||
|
||||
// Pipe the streams: Read -> Parse -> Transform -> Stringify -> Write
|
||||
inputStream
|
||||
.on('error', (error) => console.error('Input stream error:', error))
|
||||
.pipe(parse({ columns: true, trim: true }))
|
||||
.on('error', (error) => console.error('Parse error:', error))
|
||||
.pipe(transformer)
|
||||
.on('error', (error) => console.error('Transform error:', error))
|
||||
.pipe(stringify({ header: true }))
|
||||
.on('error', (error) => console.error('Stringify error:', error))
|
||||
.pipe(outputStream);
|
||||
|
||||
outputStream.on('finish', () => {
|
||||
console.log(`Successfully written to ${outputFilePath}`);
|
||||
ClosePool();
|
||||
});
|
||||
|
||||
outputStream.on('error', (error) => {
|
||||
console.error('An error occurred in the process:', error);
|
||||
ClosePool();
|
||||
});
|
||||
}
|
||||
|
||||
await PricesFromCSV();
|
||||
@@ -163,12 +163,12 @@ html {
|
||||
|
||||
.image-grow {
|
||||
transition: box-shadow 350ms ease, transform 350ms ease;
|
||||
//box-shadow: 0 2px 4px rgba(0, 0, 0, 0.24);
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.24);
|
||||
|
||||
&:hover,
|
||||
&:focus {
|
||||
box-shadow: 0 8px 10px rgba(0, 0, 0, 0.2);
|
||||
//transform: translateY(-0.9rem) scale(1.02);
|
||||
transform: translateY(-0.9rem) scale(1.02);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,12 +8,19 @@ export const relations = defineRelations(schema, (r) => ({
|
||||
to: r.skus.skuId,
|
||||
}),
|
||||
},
|
||||
salesHistory: {
|
||||
sku: r.one.skus({
|
||||
from: r.salesHistory.skuId,
|
||||
to: r.skus.skuId,
|
||||
}),
|
||||
},
|
||||
skus: {
|
||||
card: r.one.cards({
|
||||
from: [r.skus.productId, r.skus.variant],
|
||||
to: [r.cards.productId, r.cards.variant],
|
||||
}),
|
||||
history: r.many.priceHistory(),
|
||||
latestSales: r.many.salesHistory(),
|
||||
},
|
||||
cards: {
|
||||
prices: r.many.skus(),
|
||||
|
||||
@@ -109,6 +109,21 @@ export const priceHistory = pokeSchema.table('price_history', {
|
||||
primaryKey({ name: 'pk_price_history', columns: [table.skuId, table.calculatedAt] })
|
||||
]);
|
||||
|
||||
export const salesHistory = pokeSchema.table('sales_history',{
|
||||
skuId: integer().notNull(),
|
||||
orderDate: timestamp().notNull(),
|
||||
title: varchar({ length: 255 }),
|
||||
customListingId: varchar({ length: 255 }),
|
||||
language: varchar({ length: 100 }),
|
||||
listingType: varchar({ length: 100 }),
|
||||
purchasePrice: decimal({ precision: 10, scale: 2 }),
|
||||
quantity: integer(),
|
||||
shippingPrice: decimal({ precision: 10, scale: 2 })
|
||||
},
|
||||
(table) => [
|
||||
primaryKey({ name: 'pk_sales_history', columns: [table.skuId, table.orderDate] })
|
||||
]);
|
||||
|
||||
export const processingSkus = pokeSchema.table('processing_skus', {
|
||||
skuId: integer().primaryKey(),
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user