tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[chore] refactor common functions into helper script

Browse Source
This commit is contained in:
Thad Miller
2026-03-19 22:18:06 -04:00
parent 023cd87319
commit 171ce294f4
6 changed files with 160 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
scripts/indexing.ts → scripts/pokemon-helper.ts
View File

@@ -1,6 +1,9 @@
import chalk from 'chalk';
import { client } from '../src/db/typesense.ts';
import type { DBInstance } from '../src/db/index.ts';
import fs from "node:fs/promises";
import { sql } from 'drizzle-orm'
const DollarToInt = (dollar: any) => {
if (dollar === null) return null;
@@ -8,6 +11,31 @@ const DollarToInt = (dollar: any) => {
}
export const Sleep = (ms: number) => {
return new Promise(resolve => setTimeout(resolve, ms));
}
export const FileExists = async (path: string): Promise<boolean> => {
try {
await fs.access(path);
return true;
} catch {
return false;
}
}
export const GetNumberOrNull = (value: any): number | null => {
const number = Number(value); // Attempt to convert the value to a number
if (Number.isNaN(number)) {
return null; // Return null if the result is NaN
}
return number; // Otherwise, return the number
}
// Delete and recreate the 'cards' index
export const createCardCollection = async () => {
try {
@@ -102,3 +130,48 @@ export const upsertSkuCollection = async (db:DBInstance) => {
})), { action: 'upsert' });
console.log(chalk.green('Collection "skus" indexed successfully.'));
}
export const UpdateVariants = async (db:DBInstance) => {
const updates = await db.execute(sql`update cards as c
set
product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name,
sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
from (
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
) a
where c.product_id = a.product_id and c.variant = a.variant and
(
c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or
c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or
c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or
c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
)
`);
console.log(`Updated ${updates.rowCount} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name,
coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type,
coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
`);
console.log(`Inserted ${inserts.rowCount} rows into cards table`);
}

61
scripts/preload-tcgplayer.ts
View File

@@ -5,10 +5,11 @@ import { db, ClosePool } from '../src/db/index.ts';
import fs from "node:fs/promises";
import path from "node:path";
import chalk from 'chalk';
import * as helper from './pokemon-helper.ts';
//import util from 'util';
async function syncTcgplayer() {
async function syncTcgplayer(cardSets:string[] = []) {
const productLines = [ "pokemon", "pokemon-japan" ];
@@ -29,36 +30,21 @@ async function syncTcgplayer() {
const setNames = data.results[0].aggregations.setName;
for (const setName of setNames) {
let processSet = true;
if (cardSets.length > 0) {
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
}
if (processSet) {
console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
await syncProductLine(productLine, "setName", setName.urlValue);
}
}
}
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
}
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function fileExists(path: string): Promise<boolean> {
try {
await fs.access(path);
return true;
} catch {
return false;
}
}
function getNumberOrNull(value: any): number | null {
const number = Number(value); // Attempt to convert the value to a number
if (Number.isNaN(number)) {
return null; // Return null if the result is NaN
}
return number; // Otherwise, return the number
}
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
let start = 0;
@@ -123,7 +109,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
for (const item of data.results[0].results) {
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
if (allProductIds.has(item.productId)) {
if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
continue;
}
@@ -163,7 +149,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
cardTypeB: item.customAttributes.cardTypeB || null,
energyType: detailData.customAttributes.energyType?.[0] || null,
flavorText: detailData.customAttributes.flavorText || null,
hp: getNumberOrNull(item.customAttributes.hp),
hp: helper.GetNumberOrNull(item.customAttributes.hp),
number: detailData.customAttributes.number || '',
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null,
@@ -201,7 +187,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
cardTypeB: item.customAttributes.cardTypeB || null,
energyType: detailData.customAttributes.energyType?.[0] || null,
flavorText: detailData.customAttributes.flavorText || null,
hp: getNumberOrNull(item.customAttributes.hp),
hp: helper.GetNumberOrNull(item.customAttributes.hp),
number: detailData.customAttributes.number || '',
releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
resistance: item.customAttributes.resistance || null,
@@ -218,7 +204,9 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
},
});
console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
// set is...
await db.insert(schema.sets).values({
setId: detailData.setId,
@@ -255,7 +243,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
// get image if it doesn't already exist
const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
if (!await fileExists(imagePath)) {
if (!await helper.FileExists(imagePath)) {
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
if (imageResponse.ok) {
const buffer = await imageResponse.arrayBuffer();
@@ -267,7 +255,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
}
// be nice to the API and not send too many requests in a short time
await sleep(300);
await helper.Sleep(300);
}
@@ -277,8 +265,21 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
// clear the log file
await fs.rm('missing_images.log', { force: true });
let allProductIds = new Set();
const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
const args = process.argv.slice(2);
if (args.length === 0) {
allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
await syncTcgplayer();
}
else {
await syncTcgplayer(args);
}
// update the card table with new/updated variants
await helper.UpdateVariants(db);
// index the card updates
helper.upsertCardCollection(db);
await ClosePool();

6
scripts/reindex.ts
View File

@@ -1,10 +1,10 @@
import chalk from 'chalk';
import { db, ClosePool } from '../src/db/index.ts';
import * as Indexing from './indexing.ts';
import * as Indexing from './pokemon-helper.ts';
await Indexing.createCardCollection();
await Indexing.createSkuCollection();
//await Indexing.createCardCollection();
//await Indexing.createSkuCollection();
await Indexing.upsertCardCollection(db);
await Indexing.upsertSkuCollection(db);
await ClosePool();

59
scripts/sync-prices.ts
View File

@@ -3,15 +3,11 @@ import 'dotenv/config';
import chalk from 'chalk';
import { db, ClosePool } from '../src/db/index.ts';
import { sql, inArray, eq } from 'drizzle-orm';
import { skus, processingSkus, priceHistory } from '../src/db/schema.ts';
import { skus, processingSkus, priceHistory, salesHistory } from '../src/db/schema.ts';
import { toSnakeCase } from 'drizzle-orm/casing';
import * as Indexing from './indexing.ts';
import * as helper from './pokemon-helper.ts';
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function resetProcessingTable() {
// Use sql.raw to execute the TRUNCATE TABLE statement
await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;'));
@@ -21,6 +17,7 @@ async function resetProcessingTable() {
async function syncPrices() {
const batchSize = 1000;
// const skuIndex = client.collections('skus');
const updatedCards = new Set<number>();
await resetProcessingTable();
console.log(chalk.green('Processing table reset and populated with current SKUs.'));
@@ -60,7 +57,7 @@ async function syncPrices() {
// remove skus from the 'working' processingSkus table
await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
// be nice to the API and not send too many requests in a short time
await sleep(200);
await helper.Sleep(200);
continue;
}
@@ -103,21 +100,63 @@ async function syncPrices() {
});
console.log(chalk.cyan(`${skuRows.length} history rows added.`));
}
for (const productId of skuRows.filter(row => row.calculatedAt != null).map(row => row.productId)) {
updatedCards.add(productId);
}
}
// remove skus from the 'working' processingSkus table
await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
// be nice to the API and not send too many requests in a short time
await sleep(200);
await helper.Sleep(200);
}
return updatedCards;
}
const updateLatestSales = async (updatedCards: Set<number>) => {
for (const productId of updatedCards.values()) {
console.log(`Getting sale history for ${productId}`)
const salesResponse = await fetch(`https://mpapi.tcgplayer.com/v2/product/${productId}/latestsales`,{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
},
body: JSON.stringify({ conditions:[], languages:[1], limit:25, listType:"All", variants:[] }),
});
if (!salesResponse.ok) {
console.error('Error fetching sale history:', salesResponse.statusText);
process.exit(1);
}
const salesData = await salesResponse.json();
for (const sale of salesData.data) {
const skuData = await db.query.skus.findFirst({ where: { productId: productId, variant: sale.variant, condition: sale.condition } });
if (skuData) {
await db.insert(salesHistory).values({
skuId: skuData.skuId,
orderDate: new Date(sale.orderDate),
title: sale.title,
customListingId: sale.customListingId,
language: sale.language,
listingType: sale.listingType,
purchasePrice: sale.purchasePrice,
quantity: sale.quantity,
shippingPrice: sale.shippingPrice
}).onConflictDoNothing();
}
}
await helper.Sleep(500);
}
}
const start = Date.now();
await syncPrices();
await Indexing.upsertSkuCollection(db);
const updatedCards = await syncPrices();
await helper.upsertSkuCollection(db);
//console.log(updatedCards);
//console.log(updatedCards.size);
//await updateLatestSales(updatedCards);
await ClosePool();
const end = Date.now();
const duration = (end - start) / 1000;

47
scripts/sync-variants.ts
View File

@@ -1,47 +0,0 @@
import 'dotenv/config';
import { db, ClosePool } from '../src/db/index.ts';
import { sql } from 'drizzle-orm'
async function syncVariants() {
const updates = await db.execute(sql`update cards as c
set
product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name,
sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
from (
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
) a
where c.product_id = a.product_id and c.variant = a.variant and
(
c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or
c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or
c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or
c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
)
`);
console.log(`Updated ${updates.rowCount} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name,
coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type,
coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
`);
console.log(`Inserted ${inserts.rowCount} rows into cards table`);
}
await syncVariants();
await ClosePool();

2
src/db/schema.ts
View File

@@ -97,7 +97,7 @@ export const skus = pokeSchema.table('skus', {
priceCount: integer(),
},
(table) => [
index('idx_product_id_condition').on(table.productId, table.variant),
index('idx_product_id_condition').on(table.productId, table.variant, table.condition),
]);
export const priceHistory = pokeSchema.table('price_history', {