tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[feat] switched from mysql to postgresql

Browse Source
This commit is contained in:
Thad Miller
2026-03-11 19:18:45 -04:00
parent 1089bcdc20
commit a68ed7f7b8
10 changed files with 1801 additions and 1175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scripts/preload-tcgplayer.ts
View File

@@ -1,6 +1,6 @@
import 'dotenv/config';
import * as schema from '../src/db/schema.ts';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import fs from "node:fs/promises";
import path from "node:path";
@@ -43,14 +43,6 @@ function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function cleanProductName(name: string): string {
// remove TCGPlayer crap
name = name.replace(/ - .*$/, '');
name = name.replace(/ \[.*\]/, '');
name = name.replace(/ \(.*\)/, '');
return name.trim();
}
async function fileExists(path: string): Promise<boolean> {
try {
await fs.access(path);
@@ -130,10 +122,10 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
for (const item of data.results[0].results) {
// // Check if productId already exists and skip if it does (to avoid hitting the API too much)
// if (allProductIds.has(item.productId)) {
// continue;
// }
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
if (allProductIds.has(item.productId)) {
continue;
}
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
@@ -184,8 +176,9 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
medianPrice: detailData.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
}).onDuplicateKeyUpdate({
artist: detailData.formattedAttributes.Artist || null,
}).onConflictDoUpdate({
target: schema.tcgcards.productId,
set: {
productName: detailData.productName,
//productName: cleanProductName(item.productName),
@@ -221,7 +214,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
medianPrice: detailData.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
artist: detailData.formattedAttributes.Artist || null,
},
});
@@ -232,7 +225,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
setCode: detailData.setCode,
setName: detailData.setName,
setUrlName: detailData.setUrlName,
}).onDuplicateKeyUpdate({
}).onConflictDoUpdate({
target: schema.sets.setId,
set: {
setCode: detailData.setCode,
setName: detailData.setName,
@@ -249,7 +243,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
condition: skuItem.condition,
language: skuItem.language,
variant: skuItem.variant,
}).onDuplicateKeyUpdate({
}).onConflictDoUpdate({
target: schema.skus.skuId,
set: {
condition: skuItem.condition,
language: skuItem.language,
@@ -286,4 +281,4 @@ await fs.rm('missing_images.log', { force: true });
const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
await syncTcgplayer();
await poolConnection.end();
await ClosePool();

8
scripts/reindex.ts
View File

@@ -1,6 +1,6 @@
import { Client } from 'typesense';
import chalk from 'chalk';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { client } from '../src/db/typesense.ts';
import { release } from 'node:os';
@@ -95,9 +95,9 @@ async function preloadSearchIndex() {
cardType: card.cardType || "",
energyType: card.energyType || "",
number: card.number,
Artist: card.Artist || "",
Artist: card.artist || "",
sealed: card.sealed,
content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.Artist || ""].join(' '),
content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '),
releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0,
sku_id: card.prices.map(price => price.skuId.toString())
})), { action: 'upsert' });
@@ -128,7 +128,7 @@ await preloadSearchIndex().catch((error) => {
}
process.exit(1);
}).finally(() => {
poolConnection.end();
ClosePool();
console.log(chalk.blue('Database connection closed.'));
process.exit(0);
});

20
scripts/sync-prices.ts
View File

@@ -1,10 +1,11 @@
import 'dotenv/config';
import chalk from 'chalk';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { sql, inArray, eq } from 'drizzle-orm';
import { skus, processingSkus } from '../src/db/schema.ts';
import { client } from '../src/db/typesense.ts';
import { toSnakeCase } from 'drizzle-orm/casing';
const DollarToInt = (dollar: any) => {
@@ -18,7 +19,7 @@ function sleep(ms: number) {
async function resetProcessingTable() {
// Use sql.raw to execute the TRUNCATE TABLE statement
await db.execute(sql.raw('TRUNCATE TABLE processingSkus;'));
await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;'));
await db.insert(processingSkus).select(db.select({skuId: skus.skuId}).from(skus));
}
@@ -72,12 +73,13 @@ async function syncPrices() {
marketPrice: sku.marketPrice,
priceCount: null,
}});
await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({
await db.insert(skus).values(skuUpdates).onConflictDoUpdate({
target: skus.skuId,
set: {
calculatedAt: sql`values(${skus.calculatedAt})`,
highestPrice: sql`values(${skus.highestPrice})`,
lowestPrice: sql`values(${skus.lowestPrice})`,
marketPrice: sql`values(${skus.marketPrice})`,
calculatedAt: sql.raw(`excluded.${toSnakeCase(skus.calculatedAt.name)}`),
highestPrice: sql.raw(`excluded.${toSnakeCase(skus.highestPrice.name)}`),
lowestPrice: sql.raw(`excluded.${toSnakeCase(skus.lowestPrice.name)}`),
marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
}
});
@@ -85,7 +87,7 @@ async function syncPrices() {
await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
// be nice to the API and not send too many requests in a short time
await sleep(100);
await sleep(200);
}
}
@@ -106,7 +108,7 @@ async function indexPrices() {
const start = Date.now();
await syncPrices();
await indexPrices();
await poolConnection.end();
await ClosePool();
const end = Date.now();
const duration = (end - start) / 1000;
console.log(chalk.green(`Price sync completed in ${duration.toFixed(2)} seconds.`));

59
scripts/sync-variants.ts
View File

@@ -1,32 +1,47 @@
import 'dotenv/config';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { sql } from 'drizzle-orm'
async function syncVariants() {
const updates = await db.execute(sql`update cards as c
join tcgcards t on c.productId = t.productId
join (select distinct productId, variant from skus) b on c.productId = b.productId and c.variant = b.variant
left join tcg_overrides o on c.productId = o.productId
set c.productName = coalesce(o.productName, regexp_replace(regexp_replace(coalesce(nullif(t.productName, ''), t.productUrlName),' \\\\(.*\\\\)',''),' - .*$','')),
c.productLineName = coalesce(o.productLineName, t.productLineName), c.productUrlName = coalesce(o.productUrlName, t.productUrlName), c.rarityName = coalesce(o.rarityName, t.rarityName),
c.sealed = coalesce(o.sealed, t.sealed), c.setId = coalesce(o.setId, t.setId), c.cardType = coalesce(o.cardType, t.cardType),
c.energyType = coalesce(o.energyType, t.energyType), c.number = coalesce(o.number, t.number), c.Artist = coalesce(o.Artist, t.Artist)`);
console.log(`Updated ${updates[0].affectedRows} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (productId, variant, productName, productLineName, productUrlName, rarityName, sealed, setId, cardType, energyType, number, Artist)
select t.productId, b.variant,
coalesce(o.productName, regexp_replace(regexp_replace(coalesce(nullif(t.productName, ''), t.productUrlName),' \\\\(.*\\\\)',''),' - .*$','')) as productName,
coalesce(o.productLineName, t.productLineName) as productLineName, coalesce(o.productUrlName, t.productUrlName) as productUrlName, coalesce(o.rarityName, t.rarityName) as rarityName,
coalesce(o.sealed, t.sealed) as sealed, coalesce(o.setId, t.setId) as setId, coalesce(o.cardType, t.cardType) as cardType,
coalesce(o.energyType, t.energyType) as energyType, coalesce(o.number, t.number) as number, coalesce(o.Artist, t.Artist) as Artist
from tcgcards t
join (select distinct productId, variant from skus) b on t.productId = b.productId
left join tcg_overrides o on t.productId = o.productId
where not exists (select 1 from cards where productId=t.productId and variant=b.variant)
set
product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name,
sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
from (
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
) a
where c.product_id = a.product_id and c.variant = a.variant and
(
c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or
c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or
c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or
c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
)
`);
console.log(`Inserted ${inserts[0].affectedRows} rows into cards table`);
console.log(`Updated ${updates.rowCount} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name,
coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type,
coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
`);
console.log(`Inserted ${inserts.rowCount} rows into cards table`);
}
await syncVariants();
await poolConnection.end();
await ClosePool();