tmiller/pokemon
1
0
Fork 1
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Merge branch 'feat/postgresql'

Browse Source
This commit is contained in:
Thad Miller
2026-03-11 19:26:52 -04:00
parent f03c909745 a68ed7f7b8
commit c622c8bd8f
10 changed files with 1801 additions and 1175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
drizzle.config.ts
View File

@@ -4,8 +4,12 @@ import { defineConfig } from 'drizzle-kit';
export default defineConfig({
out: './drizzle', // Directory for migration files
schema: './src/db/schema.ts', // Path to your schema file
dialect: 'mysql', // Specify the database dialect
casing: 'snake_case', // camelCase JS objects become snake_case in the DB
dialect: 'postgresql', // Specify the database dialect
dbCredentials: {
url: process.env.DATABASE_URL!, // Use the URL from your .env file
},
schemaFilter: ['pokemon'],
verbose: true,
strict: true,
});

2744
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@@ -19,13 +19,14 @@
"chart.js": "^4.5.1",
"dotenv": "^17.2.4",
"drizzle-orm": "^1.0.0-beta.15-859cf75",
"mysql2": "^3.16.3",
"pg": "^8.20.0",
"sass": "^1.97.3",
"typesense": "^3.0.1"
},
"devDependencies": {
"@types/bootstrap": "^5.2.10",
"@types/node": "^25.2.1",
"@types/pg": "^8.18.0",
"drizzle-kit": "^1.0.0-beta.15-859cf75",
"typescript": "^5.9.3"
}

33
scripts/preload-tcgplayer.ts
View File

@@ -1,6 +1,6 @@
import 'dotenv/config';
import * as schema from '../src/db/schema.ts';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import fs from "node:fs/promises";
import path from "node:path";
@@ -43,14 +43,6 @@ function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function cleanProductName(name: string): string {
// remove TCGPlayer crap
name = name.replace(/ - .*$/, '');
name = name.replace(/ \[.*\]/, '');
name = name.replace(/ \(.*\)/, '');
return name.trim();
}
async function fileExists(path: string): Promise<boolean> {
try {
await fs.access(path);
@@ -130,10 +122,10 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
for (const item of data.results[0].results) {
// // Check if productId already exists and skip if it does (to avoid hitting the API too much)
// if (allProductIds.has(item.productId)) {
// continue;
// }
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
if (allProductIds.has(item.productId)) {
continue;
}
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
@@ -184,8 +176,9 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
medianPrice: detailData.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
}).onDuplicateKeyUpdate({
artist: detailData.formattedAttributes.Artist || null,
}).onConflictDoUpdate({
target: schema.tcgcards.productId,
set: {
productName: detailData.productName,
//productName: cleanProductName(item.productName),
@@ -221,7 +214,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
maxFulfillableQuantity: detailData.maxFulfillableQuantity,
medianPrice: detailData.medianPrice,
totalListings: item.totalListings,
Artist: detailData.formattedAttributes.Artist || null,
artist: detailData.formattedAttributes.Artist || null,
},
});
@@ -232,7 +225,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
setCode: detailData.setCode,
setName: detailData.setName,
setUrlName: detailData.setUrlName,
}).onDuplicateKeyUpdate({
}).onConflictDoUpdate({
target: schema.sets.setId,
set: {
setCode: detailData.setCode,
setName: detailData.setName,
@@ -249,7 +243,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
condition: skuItem.condition,
language: skuItem.language,
variant: skuItem.variant,
}).onDuplicateKeyUpdate({
}).onConflictDoUpdate({
target: schema.skus.skuId,
set: {
condition: skuItem.condition,
language: skuItem.language,
@@ -286,4 +281,4 @@ await fs.rm('missing_images.log', { force: true });
const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
await syncTcgplayer();
await poolConnection.end();
await ClosePool();

8
scripts/reindex.ts
View File

@@ -1,6 +1,6 @@
import { Client } from 'typesense';
import chalk from 'chalk';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { client } from '../src/db/typesense.ts';
import { release } from 'node:os';
@@ -95,9 +95,9 @@ async function preloadSearchIndex() {
cardType: card.cardType || "",
energyType: card.energyType || "",
number: card.number,
Artist: card.Artist || "",
Artist: card.artist || "",
sealed: card.sealed,
content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.Artist || ""].join(' '),
content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '),
releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0,
sku_id: card.prices.map(price => price.skuId.toString())
})), { action: 'upsert' });
@@ -128,7 +128,7 @@ await preloadSearchIndex().catch((error) => {
}
process.exit(1);
}).finally(() => {
poolConnection.end();
ClosePool();
console.log(chalk.blue('Database connection closed.'));
process.exit(0);
});

20
scripts/sync-prices.ts
View File

@@ -1,10 +1,11 @@
import 'dotenv/config';
import chalk from 'chalk';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { sql, inArray, eq } from 'drizzle-orm';
import { skus, processingSkus } from '../src/db/schema.ts';
import { client } from '../src/db/typesense.ts';
import { toSnakeCase } from 'drizzle-orm/casing';
const DollarToInt = (dollar: any) => {
@@ -18,7 +19,7 @@ function sleep(ms: number) {
async function resetProcessingTable() {
// Use sql.raw to execute the TRUNCATE TABLE statement
await db.execute(sql.raw('TRUNCATE TABLE processingSkus;'));
await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;'));
await db.insert(processingSkus).select(db.select({skuId: skus.skuId}).from(skus));
}
@@ -72,12 +73,13 @@ async function syncPrices() {
marketPrice: sku.marketPrice,
priceCount: null,
}});
await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({
await db.insert(skus).values(skuUpdates).onConflictDoUpdate({
target: skus.skuId,
set: {
calculatedAt: sql`values(${skus.calculatedAt})`,
highestPrice: sql`values(${skus.highestPrice})`,
lowestPrice: sql`values(${skus.lowestPrice})`,
marketPrice: sql`values(${skus.marketPrice})`,
calculatedAt: sql.raw(`excluded.${toSnakeCase(skus.calculatedAt.name)}`),
highestPrice: sql.raw(`excluded.${toSnakeCase(skus.highestPrice.name)}`),
lowestPrice: sql.raw(`excluded.${toSnakeCase(skus.lowestPrice.name)}`),
marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
}
});
@@ -85,7 +87,7 @@ async function syncPrices() {
await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
// be nice to the API and not send too many requests in a short time
await sleep(100);
await sleep(200);
}
}
@@ -106,7 +108,7 @@ async function indexPrices() {
const start = Date.now();
await syncPrices();
await indexPrices();
await poolConnection.end();
await ClosePool();
const end = Date.now();
const duration = (end - start) / 1000;
console.log(chalk.green(`Price sync completed in ${duration.toFixed(2)} seconds.`));

59
scripts/sync-variants.ts
View File

@@ -1,32 +1,47 @@
import 'dotenv/config';
import { db, poolConnection } from '../src/db/index.ts';
import { db, ClosePool } from '../src/db/index.ts';
import { sql } from 'drizzle-orm'
async function syncVariants() {
const updates = await db.execute(sql`update cards as c
join tcgcards t on c.productId = t.productId
join (select distinct productId, variant from skus) b on c.productId = b.productId and c.variant = b.variant
left join tcg_overrides o on c.productId = o.productId
set c.productName = coalesce(o.productName, regexp_replace(regexp_replace(coalesce(nullif(t.productName, ''), t.productUrlName),' \\\\(.*\\\\)',''),' - .*$','')),
c.productLineName = coalesce(o.productLineName, t.productLineName), c.productUrlName = coalesce(o.productUrlName, t.productUrlName), c.rarityName = coalesce(o.rarityName, t.rarityName),
c.sealed = coalesce(o.sealed, t.sealed), c.setId = coalesce(o.setId, t.setId), c.cardType = coalesce(o.cardType, t.cardType),
c.energyType = coalesce(o.energyType, t.energyType), c.number = coalesce(o.number, t.number), c.Artist = coalesce(o.Artist, t.Artist)`);
console.log(`Updated ${updates[0].affectedRows} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (productId, variant, productName, productLineName, productUrlName, rarityName, sealed, setId, cardType, energyType, number, Artist)
select t.productId, b.variant,
coalesce(o.productName, regexp_replace(regexp_replace(coalesce(nullif(t.productName, ''), t.productUrlName),' \\\\(.*\\\\)',''),' - .*$','')) as productName,
coalesce(o.productLineName, t.productLineName) as productLineName, coalesce(o.productUrlName, t.productUrlName) as productUrlName, coalesce(o.rarityName, t.rarityName) as rarityName,
coalesce(o.sealed, t.sealed) as sealed, coalesce(o.setId, t.setId) as setId, coalesce(o.cardType, t.cardType) as cardType,
coalesce(o.energyType, t.energyType) as energyType, coalesce(o.number, t.number) as number, coalesce(o.Artist, t.Artist) as Artist
from tcgcards t
join (select distinct productId, variant from skus) b on t.productId = b.productId
left join tcg_overrides o on t.productId = o.productId
where not exists (select 1 from cards where productId=t.productId and variant=b.variant)
set
product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name,
sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
from (
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
) a
where c.product_id = a.product_id and c.variant = a.variant and
(
c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or
c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or
c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or
c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
)
`);
console.log(`Inserted ${inserts[0].affectedRows} rows into cards table`);
console.log(`Updated ${updates.rowCount} rows in cards table`);
const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
select t.product_id, b.variant,
coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name,
coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type,
coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
from tcg_cards t
join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
left join tcg_overrides o on t.product_id = o.product_id
where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
`);
console.log(`Inserted ${inserts.rowCount} rows into cards table`);
}
await syncVariants();
await poolConnection.end();
await ClosePool();

22
src/db/index.ts
View File

@@ -1,11 +1,23 @@
// src/db/index.ts
import 'dotenv/config';
import { relations } from './relations.ts';
import { drizzle } from 'drizzle-orm/mysql2';
import mysql from 'mysql2/promise';
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
//export const poolConnection = mysql.createPool({ uri: process.env.DATABASE_URL, client_found_rows: false });
export const poolConnection = mysql.createPool({ uri: process.env.DATABASE_URL, flags: ["-FOUND_ROWS"] });
const pool = new Pool({
connectionString: process.env.DATABASE_URL,
max: 10,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
});
export const db = drizzle({ client: poolConnection, relations: relations});
// Handle pool errors to prevent connection corruption
pool.on('error', (err) => {
console.error('Unexpected error on idle client', err);
});
export const db = drizzle({ client: pool, relations: relations, casing: 'snake_case' });
export const ClosePool = () => {
pool.end();
}

79
src/db/schema.ts
View File

@@ -1,22 +1,25 @@
import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "drizzle-orm/mysql-core"
//import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "drizzle-orm/mysql-core"
import { integer, varchar, boolean, decimal, timestamp, index, pgSchema } from "drizzle-orm/pg-core";
export const tcgcards = mysqlTable("tcgcards", {
productId: int().primaryKey(),
export const pokeSchema = pgSchema("pokemon");
export const tcgcards = pokeSchema.table('tcg_cards', {
productId: integer().primaryKey(),
productName: varchar({ length: 255 }).notNull(),
productLineName: varchar({ length: 255 }).default("").notNull(),
productLineUrlName: varchar({ length: 255 }).default("").notNull(),
productStatusId: int().default(0).notNull(),
productTypeId: int().default(0).notNull(),
productStatusId: integer().default(0).notNull(),
productTypeId: integer().default(0).notNull(),
productUrlName: varchar({ length: 255 }).default("").notNull(),
rarityName: varchar({ length: 100 }).default("").notNull(),
sealed: boolean().default(false).notNull(),
sellerListable: boolean().default(false).notNull(),
setId: int(),
shippingCategoryId: int(),
setId: integer(),
shippingCategoryId: integer(),
duplicate: boolean().default(false).notNull(),
foilOnly: boolean().default(false).notNull(),
maxFulfillableQuantity: int(),
totalListings: int(),
maxFulfillableQuantity: integer(),
totalListings: integer(),
score: decimal({ precision: 10, scale: 2, mode: 'number' }),
lowestPrice: decimal({ precision: 10, scale: 2, mode: 'number' }),
lowestPriceWithShipping: decimal({ precision: 10, scale: 2, mode: 'number' }),
@@ -30,82 +33,82 @@ export const tcgcards = mysqlTable("tcgcards", {
cardTypeB: varchar({ length: 100 }),
energyType: varchar({ length: 100 }),
flavorText: varchar({ length: 1000 }),
hp: int(),
hp: integer(),
number: varchar({ length: 50 }).default("").notNull(),
releaseDate: datetime(),
releaseDate: timestamp(),
resistance: varchar({ length: 100 }),
retreatCost: varchar({ length: 100 }),
stage: varchar({ length: 100 }),
weakness: varchar({ length: 100 }),
Artist: varchar({ length: 255 }),
artist: varchar({ length: 255 }),
});
export const cards = mysqlTable("cards", {
cardId: int().notNull().primaryKey().autoincrement(),
productId: int().notNull(),
export const cards = pokeSchema.table('cards', {
cardId: integer().notNull().primaryKey().generatedAlwaysAsIdentity(),
productId: integer().notNull(),
variant: varchar({ length: 100 }).notNull(),
productName: varchar({ length: 255 }),
productLineName: varchar({ length: 255 }),
productUrlName: varchar({ length: 255 }).default("").notNull(),
rarityName: varchar({ length: 100 }),
sealed: boolean().default(false).notNull(),
setId: int(),
setId: integer(),
cardType: varchar({ length: 100 }),
energyType: varchar({ length: 100 }),
number: varchar({ length: 50 }),
Artist: varchar({ length: 255 }),
artist: varchar({ length: 255 }),
},
(table) => [
index("card_productIdIdx").on(table.productId, table.variant),
index('idx_card_product_id').on(table.productId, table.variant),
]);
export const tcg_overrides = mysqlTable("tcg_overrides", {
productId: int().primaryKey(),
export const tcg_overrides = pokeSchema.table('tcg_overrides', {
productId: integer().primaryKey(),
productName: varchar({ length: 255 }),
productLineName: varchar({ length: 255 }),
productUrlName: varchar({ length: 255 }).default("").notNull(),
productUrlName: varchar({ length: 255 }).default('').notNull(),
rarityName: varchar({ length: 100 }),
sealed: boolean().default(false).notNull(),
setId: int(),
setId: integer(),
cardType: varchar({ length: 100 }),
energyType: varchar({ length: 100 }),
number: varchar({ length: 50 }),
Artist: varchar({ length: 255 }),
artist: varchar({ length: 255 }),
});
export const sets = mysqlTable("sets", {
setId: int().primaryKey(),
export const sets = pokeSchema.table('sets', {
setId: integer().primaryKey(),
setName: varchar({ length: 255 }).notNull(),
setUrlName: varchar({ length: 255 }).notNull(),
setCode: varchar({ length: 100 }).notNull(),
});
export const skus = mysqlTable("skus", {
skuId: int().primaryKey(),
cardId: int().default(0).notNull(),
productId: int().notNull(),
export const skus = pokeSchema.table('skus', {
skuId: integer().primaryKey(),
cardId: integer().default(0).notNull(),
productId: integer().notNull(),
condition: varchar({ length: 255 }).notNull(),
language: varchar({ length: 100 }).notNull(),
variant: varchar({ length: 100 }).notNull(),
calculatedAt: datetime(),
calculatedAt: timestamp(),
highestPrice: decimal({ precision: 10, scale: 2 }),
lowestPrice: decimal({ precision: 10, scale: 2 }),
marketPrice: decimal({ precision: 10, scale: 2 }),
priceCount: int(),
priceCount: integer(),
},
(table) => [
index("productIdIdx").on(table.productId, table.variant),
index('idx_product_id_condition').on(table.productId, table.variant),
]);
export const priceHistory = mysqlTable("price_history", {
skuId: int().default(0).notNull(),
calculatedAt: datetime(),
export const priceHistory = pokeSchema.table('price_history', {
skuId: integer().default(0).notNull(),
calculatedAt: timestamp(),
marketPrice: decimal({ precision: 10, scale: 2 }),
},
(table) => [
index("idx_price_history").on(table.skuId, table.calculatedAt),
index('idx_price_history').on(table.skuId, table.calculatedAt),
]);
export const processingSkus = mysqlTable("processingSkus", {
skuId: int().primaryKey(),
export const processingSkus = pokeSchema.table('processing_skus', {
skuId: integer().primaryKey(),
});

2
src/pages/partials/card-modal.astro
View File

@@ -137,7 +137,7 @@ const altSearchUrl = (card: any) => {
<div class="position-relative mt-1"><img src={`/cards/${card?.productId}.jpg`} class="card-image w-100 img-fluid rounded-4" alt={card?.productName} onerror="this.onerror=null;this.src='/cards/default.jpg'" onclick="copyImage(this); dataLayer.push({'event': 'copiedImage'});"><span class="position-absolute top-50 start-0 d-inline"><FirstEditionIcon edition={card?.variant} /></span><span class="position-absolute bottom-0 start-0 d-inline"><SetIcon set={card?.set?.setCode} /></span><span class="position-absolute top-0 end-0 d-inline"><EnergyIcon energy={card?.energyType} /></span><span class="rarity-icon-large position-absolute bottom-0 end-0 d-inline"><RarityIcon rarity={card?.rarityName} /></span></div>
<div class="d-flex flex-column flex-lg-row justify-content-between mt-2">
<div class="text-secondary">{card?.set?.setCode}</div>
<div class="text-secondary">Illus<span class="d-none d-lg-inline">trator</span>: {card?.Artist}</div>
<div class="text-secondary">Illus<span class="d-none d-lg-inline">trator</span>: {card?.artist}</div>
</div>
</div>
<div class="col-sm-12 col-md-7">