[feat] switched from mysql to postgresql

drizzle.config.ts

@@ -4,8 +4,12 @@ import { defineConfig } from 'drizzle-kit';
 export default defineConfig({
   out: './drizzle', // Directory for migration files
   schema: './src/db/schema.ts', // Path to your schema file
-  dialect: 'mysql', // Specify the database dialect
+  casing: 'snake_case', // camelCase JS objects become snake_case in the DB
+  dialect: 'postgresql', // Specify the database dialect
   dbCredentials: {
     url: process.env.DATABASE_URL!, // Use the URL from your .env file
   },
+  schemaFilter: ['pokemon'],
+  verbose: true,
+  strict: true,
 });

package.json

@@ -19,13 +19,14 @@
     "chart.js": "^4.5.1",
     "dotenv": "^17.2.4",
     "drizzle-orm": "^1.0.0-beta.15-859cf75",
-    "mysql2": "^3.16.3",
+    "pg": "^8.20.0",
     "sass": "^1.97.3",
     "typesense": "^3.0.1"
   },
   "devDependencies": {
     "@types/bootstrap": "^5.2.10",
     "@types/node": "^25.2.1",
+    "@types/pg": "^8.18.0",
     "drizzle-kit": "^1.0.0-beta.15-859cf75",
     "typescript": "^5.9.3"
   }

scripts/preload-tcgplayer.ts

@@ -1,6 +1,6 @@
 import 'dotenv/config';
 import * as schema from '../src/db/schema.ts';
-import { db, poolConnection } from '../src/db/index.ts';
+import { db, ClosePool } from '../src/db/index.ts';
 
 import fs from "node:fs/promises";
 import path from "node:path";
@@ -43,14 +43,6 @@ function sleep(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
-function cleanProductName(name: string): string {
-  // remove TCGPlayer crap
-  name = name.replace(/ - .*$/, '');
-  name = name.replace(/ \[.*\]/, '');
-  name = name.replace(/ \(.*\)/, '');
-  return name.trim();
-}
-
 async function fileExists(path: string): Promise<boolean> {
   try {
     await fs.access(path);
@@ -130,10 +122,10 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
 
     for (const item of data.results[0].results) {
 
-      // // Check if productId already exists and skip if it does (to avoid hitting the API too much)
+      // Check if productId already exists and skip if it does (to avoid hitting the API too much)
-      // if (allProductIds.has(item.productId)) {
+      if (allProductIds.has(item.productId)) {
-      //   continue;
+        continue;
-      // }
+      }
 
       console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
 
@@ -184,8 +176,9 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         maxFulfillableQuantity: detailData.maxFulfillableQuantity,
         medianPrice: detailData.medianPrice,
         totalListings: item.totalListings,
-        Artist: detailData.formattedAttributes.Artist || null,
+        artist: detailData.formattedAttributes.Artist || null,
-      }).onDuplicateKeyUpdate({
+      }).onConflictDoUpdate({
+        target: schema.tcgcards.productId,
         set: {
           productName: detailData.productName,
           //productName: cleanProductName(item.productName),
@@ -221,7 +214,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
           maxFulfillableQuantity: detailData.maxFulfillableQuantity,
           medianPrice: detailData.medianPrice,
           totalListings: item.totalListings,
-          Artist: detailData.formattedAttributes.Artist || null,
+          artist: detailData.formattedAttributes.Artist || null,
         },
       });
 
@@ -232,7 +225,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
         setCode: detailData.setCode,
         setName: detailData.setName,
         setUrlName: detailData.setUrlName,
-      }).onDuplicateKeyUpdate({
+      }).onConflictDoUpdate({
+        target: schema.sets.setId,
         set: {
           setCode: detailData.setCode,
           setName: detailData.setName,
@@ -249,7 +243,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
           condition: skuItem.condition,
           language: skuItem.language,
           variant: skuItem.variant,
-        }).onDuplicateKeyUpdate({
+        }).onConflictDoUpdate({
+          target: schema.skus.skuId,
           set: {
             condition: skuItem.condition,
             language: skuItem.language,
@@ -286,4 +281,4 @@ await fs.rm('missing_images.log', { force: true });
 const allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
 
 await syncTcgplayer();
-await poolConnection.end();
+await ClosePool();

scripts/reindex.ts

@@ -1,6 +1,6 @@
 import { Client } from 'typesense';
 import chalk from 'chalk';
-import { db, poolConnection } from '../src/db/index.ts';
+import { db, ClosePool } from '../src/db/index.ts';
 import { client } from '../src/db/typesense.ts';
 import { release } from 'node:os';
 
@@ -95,9 +95,9 @@ async function preloadSearchIndex() {
         cardType: card.cardType || "",
         energyType: card.energyType || "",
         number: card.number,
-        Artist: card.Artist || "",
+        Artist: card.artist || "",
         sealed: card.sealed,
-        content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.Artist || ""].join(' '),
+        content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '),
         releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0,
         sku_id: card.prices.map(price => price.skuId.toString())
     })), { action: 'upsert' });
@@ -128,7 +128,7 @@ await preloadSearchIndex().catch((error) => {
     }
     process.exit(1);
 }).finally(() => {
-    poolConnection.end();
+    ClosePool();
     console.log(chalk.blue('Database connection closed.'));
     process.exit(0);
 });

scripts/sync-prices.ts

@@ -1,10 +1,11 @@
 import 'dotenv/config';
 
 import chalk from 'chalk';
-import { db, poolConnection } from '../src/db/index.ts';
+import { db, ClosePool } from '../src/db/index.ts';
 import { sql, inArray, eq } from 'drizzle-orm';
 import { skus, processingSkus } from '../src/db/schema.ts';
 import { client } from '../src/db/typesense.ts';
+import { toSnakeCase } from 'drizzle-orm/casing';
 
 
 const DollarToInt = (dollar: any) => {
@@ -18,7 +19,7 @@ function sleep(ms: number) {
 
 async function resetProcessingTable() {
     // Use sql.raw to execute the TRUNCATE TABLE statement
-    await db.execute(sql.raw('TRUNCATE TABLE processingSkus;'));
+    await db.execute(sql.raw('TRUNCATE TABLE pokemon.processing_skus;'));
     await db.insert(processingSkus).select(db.select({skuId: skus.skuId}).from(skus));
 }
 
@@ -72,12 +73,13 @@ async function syncPrices() {
             marketPrice: sku.marketPrice,
             priceCount: null,
         }});
-        await db.insert(skus).values(skuUpdates).onDuplicateKeyUpdate({
+        await db.insert(skus).values(skuUpdates).onConflictDoUpdate({
+            target: skus.skuId,
             set: {
-                calculatedAt: sql`values(${skus.calculatedAt})`,
+                calculatedAt: sql.raw(`excluded.${toSnakeCase(skus.calculatedAt.name)}`),
-                highestPrice: sql`values(${skus.highestPrice})`,
+                highestPrice: sql.raw(`excluded.${toSnakeCase(skus.highestPrice.name)}`),
-                lowestPrice: sql`values(${skus.lowestPrice})`,
+                lowestPrice: sql.raw(`excluded.${toSnakeCase(skus.lowestPrice.name)}`),
-                marketPrice: sql`values(${skus.marketPrice})`,
+                marketPrice: sql.raw(`excluded.${toSnakeCase(skus.marketPrice.name)}`),
             }
         });
 
@@ -85,7 +87,7 @@ async function syncPrices() {
         await db.delete(processingSkus).where(inArray(processingSkus.skuId, skuIds));
 
         // be nice to the API and not send too many requests in a short time
-        await sleep(100);
+        await sleep(200);
     }
 
 }
@@ -106,7 +108,7 @@ async function indexPrices() {
 const start = Date.now();
 await syncPrices();
 await indexPrices();
-await poolConnection.end();
+await ClosePool();
 const end = Date.now();
 const duration = (end - start) / 1000;
 console.log(chalk.green(`Price sync completed in ${duration.toFixed(2)} seconds.`));

scripts/sync-variants.ts

@@ -1,32 +1,47 @@
 import 'dotenv/config';
-import { db, poolConnection } from '../src/db/index.ts';
+import { db, ClosePool } from '../src/db/index.ts';
 import { sql } from 'drizzle-orm'
 
 async function syncVariants() {
     const updates = await db.execute(sql`update cards as c
-join tcgcards t on c.productId = t.productId
+set 
-join (select distinct productId, variant from skus) b on c.productId = b.productId and c.variant = b.variant
+    product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name, 
-left join tcg_overrides o on c.productId = o.productId
+    sealed = a.sealed, set_id = a.set_id, card_type = a.card_type, energy_type = a.energy_type, number = a.number, artist = a.artist
-set c.productName = coalesce(o.productName, regexp_replace(regexp_replace(coalesce(nullif(t.productName, ''), t.productUrlName),' \\\\(.*\\\\)',''),' - .*$','')),
+from (
-c.productLineName = coalesce(o.productLineName, t.productLineName), c.productUrlName = coalesce(o.productUrlName, t.productUrlName), c.rarityName = coalesce(o.rarityName, t.rarityName), 
+    select t.product_id, b.variant,
-c.sealed = coalesce(o.sealed, t.sealed), c.setId = coalesce(o.setId, t.setId), c.cardType = coalesce(o.cardType, t.cardType), 
+        coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
-c.energyType = coalesce(o.energyType, t.energyType), c.number = coalesce(o.number, t.number), c.Artist = coalesce(o.Artist, t.Artist)`);
+        coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name,
-    console.log(`Updated ${updates[0].affectedRows} rows in cards table`);
+        coalesce(o.rarity_name, t.rarity_name) as rarity_name, coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id,
-
+        coalesce(o.card_type, t.card_type) as card_type, coalesce(o.energy_type, t.energy_type) as energy_type,
-    const inserts = await db.execute(sql`insert into cards (productId, variant, productName, productLineName, productUrlName, rarityName, sealed, setId, cardType, energyType, number, Artist)
+        coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
-select t.productId, b.variant, 
+    from tcg_cards t
-coalesce(o.productName, regexp_replace(regexp_replace(coalesce(nullif(t.productName, ''), t.productUrlName),' \\\\(.*\\\\)',''),' - .*$','')) as productName,
+    join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
-coalesce(o.productLineName, t.productLineName) as productLineName, coalesce(o.productUrlName, t.productUrlName) as productUrlName, coalesce(o.rarityName, t.rarityName) as rarityName, 
+    left join tcg_overrides o on t.product_id = o.product_id
-coalesce(o.sealed, t.sealed) as sealed, coalesce(o.setId, t.setId) as setId, coalesce(o.cardType, t.cardType) as cardType, 
+    ) a
-coalesce(o.energyType, t.energyType) as energyType, coalesce(o.number, t.number) as number, coalesce(o.Artist, t.Artist) as Artist
+where c.product_id = a.product_id and c.variant = a.variant and
-from tcgcards t
+(
-join (select distinct productId, variant from skus) b on t.productId = b.productId
+    c.product_name is distinct from a.product_name or c.product_line_name is distinct from a.product_line_name or 
-left join tcg_overrides o on t.productId = o.productId
+	c.product_url_name is distinct from a.product_url_name or c.rarity_name is distinct from a.rarity_name or 
-where not exists (select 1 from cards where productId=t.productId and variant=b.variant)
+    c.sealed is distinct from a.sealed or c.set_id is distinct from a.set_id or c.card_type is distinct from a.card_type or 
+	c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
+)
 `);
-    console.log(`Inserted ${inserts[0].affectedRows} rows into cards table`);
+    console.log(`Updated ${updates.rowCount} rows in cards table`);
+
+    const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
+select t.product_id, b.variant, 
+coalesce(o.product_name, regexp_replace(regexp_replace(coalesce(nullif(t.product_name, ''), t.product_url_name),' \\\\(.*\\\\)',''),' - .*$','')) as product_name,
+coalesce(o.product_line_name, t.product_line_name) as product_line_name, coalesce(o.product_url_name, t.product_url_name) as product_url_name, coalesce(o.rarity_name, t.rarity_name) as rarity_name, 
+coalesce(o.sealed, t.sealed) as sealed, coalesce(o.set_id, t.set_id) as set_id, coalesce(o.card_type, t.card_type) as card_type, 
+coalesce(o.energy_type, t.energy_type) as energy_type, coalesce(o.number, t.number) as number, coalesce(o.artist, t.artist) as artist
+from tcg_cards t
+join (select distinct product_id, variant from skus) b on t.product_id = b.product_id
+left join tcg_overrides o on t.product_id = o.product_id
+where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
+`);
+    console.log(`Inserted ${inserts.rowCount} rows into cards table`);
 
 }
 
 await syncVariants();
-await poolConnection.end();
+await ClosePool();

src/db/index.ts

@@ -1,11 +1,23 @@
 // src/db/index.ts
 import 'dotenv/config';
 import { relations } from './relations.ts';
-import { drizzle } from 'drizzle-orm/mysql2';
+import { drizzle } from "drizzle-orm/node-postgres";
-import mysql from 'mysql2/promise';
+import { Pool } from "pg";
 
-//export const poolConnection = mysql.createPool({ uri: process.env.DATABASE_URL, client_found_rows: false });
+const pool = new Pool({
-export const poolConnection = mysql.createPool({ uri: process.env.DATABASE_URL, flags: ["-FOUND_ROWS"] });
+  connectionString: process.env.DATABASE_URL,
+  max: 10,
+  idleTimeoutMillis: 30000,
+  connectionTimeoutMillis: 2000,
+});
 
-export const db = drizzle({ client: poolConnection, relations: relations});
+// Handle pool errors to prevent connection corruption
+pool.on('error', (err) => {
+  console.error('Unexpected error on idle client', err);
+});
 
+export const db = drizzle({ client: pool, relations: relations, casing: 'snake_case' });
+
+export const ClosePool = () => {
+    pool.end();
+}

src/db/schema.ts

@@ -1,22 +1,25 @@
-import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "drizzle-orm/mysql-core"
+//import { mysqlTable, int, varchar, boolean, decimal, datetime, index } from "drizzle-orm/mysql-core"
+import { integer, varchar, boolean, decimal, timestamp, index, pgSchema } from "drizzle-orm/pg-core";
 
-export const tcgcards = mysqlTable("tcgcards", {
+export const pokeSchema = pgSchema("pokemon");
-	productId: int().primaryKey(),
+
+export const tcgcards = pokeSchema.table('tcg_cards', {
+	productId: integer().primaryKey(),
 	productName: varchar({ length: 255 }).notNull(),
 	productLineName: varchar({ length: 255 }).default("").notNull(),
 	productLineUrlName: varchar({ length: 255 }).default("").notNull(),
-	productStatusId: int().default(0).notNull(),
+	productStatusId: integer().default(0).notNull(),
-	productTypeId: int().default(0).notNull(),
+	productTypeId: integer().default(0).notNull(),
 	productUrlName: varchar({ length: 255 }).default("").notNull(),
 	rarityName: varchar({ length: 100 }).default("").notNull(),
 	sealed: boolean().default(false).notNull(),
 	sellerListable: boolean().default(false).notNull(),
-	setId: int(),
+	setId: integer(),
-	shippingCategoryId: int(),
+	shippingCategoryId: integer(),
 	duplicate: boolean().default(false).notNull(),
 	foilOnly: boolean().default(false).notNull(),
-	maxFulfillableQuantity: int(),
+	maxFulfillableQuantity: integer(),
-	totalListings: int(),
+	totalListings: integer(),
 	score: decimal({ precision: 10, scale: 2, mode: 'number' }),
 	lowestPrice: decimal({ precision: 10, scale: 2, mode: 'number' }),
 	lowestPriceWithShipping: decimal({ precision: 10, scale: 2, mode: 'number' }),
@@ -30,82 +33,82 @@ export const tcgcards = mysqlTable("tcgcards", {
 	cardTypeB: varchar({ length: 100 }),
 	energyType: varchar({ length: 100 }),
 	flavorText: varchar({ length: 1000 }),
-	hp: int(),
+	hp: integer(),
 	number: varchar({ length: 50 }).default("").notNull(),
-	releaseDate: datetime(),
+	releaseDate: timestamp(),
 	resistance: varchar({ length: 100 }),
 	retreatCost: varchar({ length: 100 }),
 	stage: varchar({ length: 100 }),
 	weakness: varchar({ length: 100 }),
-	Artist: varchar({ length: 255 }),
+	artist: varchar({ length: 255 }),
 });
 
-export const cards = mysqlTable("cards", {
+export const cards = pokeSchema.table('cards', {
-	cardId: int().notNull().primaryKey().autoincrement(),
+	cardId: integer().notNull().primaryKey().generatedAlwaysAsIdentity(),
-	productId: int().notNull(),
+	productId: integer().notNull(),
 	variant: varchar({ length: 100 }).notNull(),
 	productName: varchar({ length: 255 }),
 	productLineName: varchar({ length: 255 }),
 	productUrlName: varchar({ length: 255 }).default("").notNull(),
 	rarityName: varchar({ length: 100 }),
 	sealed: boolean().default(false).notNull(),
-	setId: int(),
+	setId: integer(),
 	cardType: varchar({ length: 100 }),
 	energyType: varchar({ length: 100 }),
 	number: varchar({ length: 50 }),
-	Artist: varchar({ length: 255 }),
+	artist: varchar({ length: 255 }),
 },
 (table) => [
-	index("card_productIdIdx").on(table.productId, table.variant),
+	index('idx_card_product_id').on(table.productId, table.variant),
 ]);
 
-export const tcg_overrides = mysqlTable("tcg_overrides", {
+export const tcg_overrides = pokeSchema.table('tcg_overrides', {
-	productId: int().primaryKey(),
+	productId: integer().primaryKey(),
 	productName: varchar({ length: 255 }),
 	productLineName: varchar({ length: 255 }),
-	productUrlName: varchar({ length: 255 }).default("").notNull(),
+	productUrlName: varchar({ length: 255 }).default('').notNull(),
 	rarityName: varchar({ length: 100 }),
 	sealed: boolean().default(false).notNull(),
-	setId: int(),
+	setId: integer(),
 	cardType: varchar({ length: 100 }),
 	energyType: varchar({ length: 100 }),
 	number: varchar({ length: 50 }),
-	Artist: varchar({ length: 255 }),
+	artist: varchar({ length: 255 }),
 });
 
-export const sets = mysqlTable("sets", {
+export const sets = pokeSchema.table('sets', {
-	setId: int().primaryKey(),
+	setId: integer().primaryKey(),
 	setName: varchar({ length: 255 }).notNull(),
 	setUrlName: varchar({ length: 255 }).notNull(),
 	setCode: varchar({ length: 100 }).notNull(),
 });
 
-export const skus = mysqlTable("skus", {
+export const skus = pokeSchema.table('skus', {
-	skuId: int().primaryKey(),
+	skuId: integer().primaryKey(),
-	cardId: int().default(0).notNull(),
+	cardId: integer().default(0).notNull(),
-	productId: int().notNull(),
+	productId: integer().notNull(),
 	condition: varchar({ length: 255 }).notNull(),
 	language: varchar({ length: 100 }).notNull(),
 	variant: varchar({ length: 100 }).notNull(),
-	calculatedAt: datetime(),
+	calculatedAt: timestamp(),
 	highestPrice: decimal({ precision: 10, scale: 2 }),
 	lowestPrice: decimal({ precision: 10, scale: 2 }),
 	marketPrice: decimal({ precision: 10, scale: 2 }),
-	priceCount: int(),
+	priceCount: integer(),
 },
 (table) => [
-	index("productIdIdx").on(table.productId, table.variant),
+	index('idx_product_id_condition').on(table.productId, table.variant),
 ]);
 
-export const priceHistory = mysqlTable("price_history", {
+export const priceHistory = pokeSchema.table('price_history', {
-	skuId: int().default(0).notNull(),
+	skuId: integer().default(0).notNull(),
-	calculatedAt: datetime(),
+	calculatedAt: timestamp(),
 	marketPrice: decimal({ precision: 10, scale: 2 }),
 },
 (table) => [
-	index("idx_price_history").on(table.skuId, table.calculatedAt),
+	index('idx_price_history').on(table.skuId, table.calculatedAt),
 ]);
 
-export const processingSkus = mysqlTable("processingSkus", {
+export const processingSkus = pokeSchema.table('processing_skus', {
-	skuId: int().primaryKey(),
+	skuId: integer().primaryKey(),
 });

src/pages/partials/card-modal.astro

@@ -147,7 +147,7 @@ const ebaySearchUrl = (card: any) => {
                     <div class="position-relative mt-1"><img src={`/cards/${card?.productId}.jpg`} class="card-image w-100 img-fluid rounded-4" alt={card?.productName} onerror="this.onerror=null;this.src='/cards/default.jpg'" onclick="copyImage(this); dataLayer.push({'event': 'copiedImage'});"><span class="position-absolute top-50 start-0 d-inline"><FirstEditionIcon edition={card?.variant} /></span><span class="position-absolute bottom-0 start-0 d-inline"><SetIcon set={card?.set?.setCode} /></span><span class="position-absolute top-0 end-0 d-inline"><EnergyIcon energy={card?.energyType} /></span><span class="rarity-icon-large position-absolute bottom-0 end-0 d-inline"><RarityIcon rarity={card?.rarityName} /></span></div>
                     <div class="d-flex flex-column flex-lg-row justify-content-between mt-2">
                     <div class="text-secondary">{card?.set?.setCode}</div>
-                    <div class="text-secondary">Illus<span class="d-none d-lg-inline">trator</span>: {card?.Artist}</div>
+                    <div class="text-secondary">Illus<span class="d-none d-lg-inline">trator</span>: {card?.artist}</div>
                     </div>
                     </div>
                 <div class="col-sm-12 col-md-7">