[chore] refactor indexing scripts

scripts/indexing.ts

@@ -0,0 +1,99 @@
+import chalk from 'chalk';
+import { client } from '../src/db/typesense.ts';
+import type { DBInstance } from '../src/db/index.ts';
+
+const DollarToInt = (dollar: any) => {
+    if (dollar === null) return null;
+    return Math.round(dollar * 100);
+}
+
+
+// Delete and recreate the 'cards' index
+export const createCardCollection = async () => {
+    try {
+        await client.collections('cards').delete();
+    } catch (error) {
+        // Ignore error, just means collection doesn't exist
+    }
+    await client.collections().create({
+        name: 'cards',
+        fields: [
+            { name: 'id', type: 'string' },
+            { name: 'cardId', type: 'int32' },
+            { name: 'productId', type: 'int32' },
+            { name: 'variant', type: 'string', facet: true },
+            { name: 'productName', type: 'string' },
+            { name: 'productLineName', type: 'string', facet: true },
+            { name: 'rarityName', type: 'string', facet: true },
+            { name: 'setName', type: 'string', facet: true },
+            { name: 'cardType', type: 'string', facet: true },
+            { name: 'energyType', type: 'string', facet: true },
+            { name: 'number', type: 'string', sort: true },
+            { name: 'Artist', type: 'string' },
+            { name: 'sealed', type: 'bool' },
+            { name: 'releaseDate', type: 'int32'},
+            { name: 'content', type: 'string', token_separators: ['/'] },
+            { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true }
+        ],
+        //default_sorting_field: 'productId',
+    });
+    console.log(chalk.green('Collection "cards" created successfully.'));
+}
+
+// Delete and recreate the 'skus' index
+export const createSkuCollection = async () => {
+    try {
+        await client.collections('skus').delete();
+    } catch (error) {
+        // Ignore error, just means collection doesn't exist
+    }
+    await client.collections().create({
+        name: 'skus',
+        fields: [
+            { name: 'id', type: 'string' },
+            { name: 'condition', type: 'string' },
+            { name: 'highestPrice', type: 'int32', optional: true },
+            { name: 'lowestPrice', type: 'int32', optional: true },
+            { name: 'marketPrice', type: 'int32', optional: true },
+        ]
+    });
+    console.log(chalk.green('Collection "skus" created successfully.'));
+}
+
+
+export const upsertCardCollection = async (db:DBInstance) => {
+    const pokemon = await db.query.cards.findMany({
+        with: { set: true, tcgdata: true, prices: true },
+    });
+    await client.collections('cards').documents().import(pokemon.map(card => ({
+        id: card.cardId.toString(),
+        cardId: card.cardId,
+        productId: card.productId,
+        variant: card.variant,
+        productName: card.productName,
+        productLineName: card.productLineName,
+        rarityName: card.rarityName,
+        setName: card.set?.setName || "",
+        cardType: card.cardType || "",
+        energyType: card.energyType || "",
+        number: card.number,
+        Artist: card.artist || "",
+        sealed: card.sealed,
+        content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '),
+        releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0,
+        sku_id: card.prices.map(price => price.skuId.toString())
+    })), { action: 'upsert' });
+    console.log(chalk.green('Collection "cards" indexed successfully.'));
+}
+
+export const upsertSkuCollection = async (db:DBInstance) => {
+    const skus = await db.query.skus.findMany();
+    await client.collections('skus').documents().import(skus.map(sku => ({
+        id: sku.skuId.toString(),
+        condition: sku.condition,
+        highestPrice: DollarToInt(sku.highestPrice),
+        lowestPrice: DollarToInt(sku.lowestPrice),
+        marketPrice: DollarToInt(sku.marketPrice),
+    })), { action: 'upsert' });
+    console.log(chalk.green('Collection "skus" indexed successfully.'));
+}

scripts/reindex.ts

@@ -1,134 +1,11 @@
-import { Client } from 'typesense';
 import chalk from 'chalk';
 import { db, ClosePool } from '../src/db/index.ts';
-import { client } from '../src/db/typesense.ts';
+import * as Indexing from './indexing.ts';
-import { release } from 'node:os';
-
-const DollarToInt = (dollar: any) => {
-    if (dollar === null) return null;
-    return Math.round(dollar * 100);
-}
-
-async function createCollection(client: Client) {
-    // Delete the collection if it already exists to ensure a clean slate
-    try {
-        await client.collections('cards').delete();
-        await client.collections('skus').delete();
-        //console.log(`Collection "cards" deleted successfully:`, response);
-    } catch (error) {
-        //console.error(`Error deleting collection "cards":`, error);
-    }
-
-    // Create the collection with the specified schema
-    try {
-        await client.collections('cards').retrieve();
-        console.log(chalk.yellow('Collection "cards" already exists.'));
-    } catch (error) {
-        if (error instanceof Error && error.message.includes('404')) {
-            await client.collections().create({
-                name: 'cards',
-                fields: [
-                    { name: 'id', type: 'string' },
-                    { name: 'cardId', type: 'int32' },
-                    { name: 'productId', type: 'int32' },
-                    { name: 'variant', type: 'string', facet: true },
-                    { name: 'productName', type: 'string' },
-                    { name: 'productLineName', type: 'string', facet: true },
-                    { name: 'rarityName', type: 'string', facet: true },
-                    { name: 'setName', type: 'string', facet: true },
-                    { name: 'cardType', type: 'string', facet: true },
-                    { name: 'energyType', type: 'string', facet: true },
-                    { name: 'number', type: 'string', sort: true },
-                    { name: 'Artist', type: 'string' },
-                    { name: 'sealed', type: 'bool' },
-                    { name: 'releaseDate', type: 'int32'},
-                    { name: 'content', type: 'string', token_separators: ['/'] },
-                    { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true }
-                ],
-                //default_sorting_field: 'productId',
-            });
-            console.log(chalk.green('Collection "cards" created successfully.'));
-        } else {
-            console.error(chalk.red('Error checking/creating collection:'), error);
-            process.exit(1);
-        }
-    }
-
-    try {
-        await client.collections('skus').retrieve();
-        console.log(chalk.yellow('Collection "skus" already exists.'));
-    } catch(error) {
-        if (error instanceof Error && error.message.includes('404')) {
-            await client.collections().create({
-                name: 'skus',
-                fields: [
-                    { name: 'id', type: 'string' },
-                    { name: 'condition', type: 'string' },
-                    { name: 'highestPrice', type: 'int32', optional: true },
-                    { name: 'lowestPrice', type: 'int32', optional: true },
-                    { name: 'marketPrice', type: 'int32', optional: true },
-                    //{ name: 'card_id', type: 'string', reference: 'cards.id' },
-                ]
-            });
-        }
-    }
-}
 
 
-async function preloadSearchIndex() {
+await Indexing.createCardCollection();
-    const pokemon = await db.query.cards.findMany({
+await Indexing.createSkuCollection();
-        with: { set: true, tcgdata: true, prices: true },
+await Indexing.upsertCardCollection(db);
-    });
+await Indexing.upsertSkuCollection(db);
-
+await ClosePool();
-    // Ensure the collection exists before importing documents
+console.log(chalk.green('Pokémon reindex complete.'));
-    await createCollection(client);
-
-    await client.collections('cards').documents().import(pokemon.map(card => ({
-        id: card.cardId.toString(),
-        cardId: card.cardId,
-        productId: card.productId,
-        variant: card.variant,
-        productName: card.productName,
-        productLineName: card.productLineName,
-        rarityName: card.rarityName,
-        setName: card.set?.setName || "",
-        cardType: card.cardType || "",
-        energyType: card.energyType || "",
-        number: card.number,
-        Artist: card.artist || "",
-        sealed: card.sealed,
-        content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '),
-        releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0,
-        sku_id: card.prices.map(price => price.skuId.toString())
-    })), { action: 'upsert' });
-
-    const skus = await db.query.skus.findMany({
-        with: { card: true }
-    });
-
-    await client.collections('skus').documents().import(skus.map(sku => ({
-        id: sku.skuId.toString(),
-        condition: sku.condition,
-        highestPrice: DollarToInt(sku.highestPrice),
-        lowestPrice: DollarToInt(sku.lowestPrice),
-        marketPrice: DollarToInt(sku.marketPrice),
-        //card_id: sku.card?.cardId.toString()
-    })));
-
-    console.log(chalk.green('Search index preloaded with Pokémon cards.'));
-
-}
-
-await preloadSearchIndex().catch((error) => {
-    console.error(chalk.red('Error preloading search index:'), error);
-    for (const e of error.importResults) {
-        if (!e.success) {
-            console.error(chalk.red(`Error importing document ${e.id}:`), e.error);
-        }
-    }
-    process.exit(1);
-}).finally(() => {
-    ClosePool();
-    console.log(chalk.blue('Database connection closed.'));
-    process.exit(0);
-});

scripts/sync-prices.ts

@@ -3,16 +3,11 @@ import 'dotenv/config';
 import chalk from 'chalk';
 import { db, ClosePool } from '../src/db/index.ts';
 import { sql, inArray, eq } from 'drizzle-orm';
-import { skus, processingSkus } from '../src/db/schema.ts';
+import { skus, processingSkus, priceHistory } from '../src/db/schema.ts';
-import { client } from '../src/db/typesense.ts';
 import { toSnakeCase } from 'drizzle-orm/casing';
+import * as Indexing from './indexing.ts';
 
 
-const DollarToInt = (dollar: any) => {
-    if (dollar === null) return null;
-    return Math.round(dollar * 100);
-}
-
 function sleep(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
@@ -73,7 +68,7 @@ async function syncPrices() {
             marketPrice: sku.marketPrice,
             priceCount: null,
         }});
-        await db.insert(skus).values(skuUpdates).onConflictDoUpdate({
+        const skuRows = await db.insert(skus).values(skuUpdates).onConflictDoUpdate({
             target: skus.skuId,
             set: {
                 calculatedAt: sql.raw(`excluded.${toSnakeCase(skus.calculatedAt.name)}`),
@@ -92,22 +87,10 @@ async function syncPrices() {
 
 }
 
-async function indexPrices() {
-    const skus = await db.query.skus.findMany();
-
-    await client.collections('skus').documents().import(skus.map(sku => ({
-        id: sku.skuId.toString(),
-        condition: sku.condition,
-        highestPrice: DollarToInt(sku.highestPrice),
-        lowestPrice: DollarToInt(sku.lowestPrice),
-        marketPrice: DollarToInt(sku.marketPrice),
-    })), { action: 'upsert' });
-
-}
 
 const start = Date.now();
 await syncPrices();
-await indexPrices();
+await Indexing.upsertSkuCollection(db);
 await ClosePool();
 const end = Date.now();
 const duration = (end - start) / 1000;

src/db/index.ts

@@ -17,6 +17,7 @@ pool.on('error', (err) => {
 });
 
 export const db = drizzle({ client: pool, relations: relations, casing: 'snake_case' });
+export type DBInstance = typeof db;
 
 export const ClosePool = () => {
     pool.end();