From 485f26de7b2117d5d0a9f01905af735850e08ba0 Mon Sep 17 00:00:00 2001 From: Thad Miller Date: Thu, 12 Mar 2026 08:18:40 -0400 Subject: [PATCH] [chore] refactor indexing scripts --- scripts/indexing.ts | 99 +++++++++++++++++++++++++++++ scripts/reindex.ts | 137 +++-------------------------------------- scripts/sync-prices.ts | 25 ++------ src/db/index.ts | 1 + 4 files changed, 111 insertions(+), 151 deletions(-) create mode 100644 scripts/indexing.ts diff --git a/scripts/indexing.ts b/scripts/indexing.ts new file mode 100644 index 0000000..2aaf4e6 --- /dev/null +++ b/scripts/indexing.ts @@ -0,0 +1,99 @@ +import chalk from 'chalk'; +import { client } from '../src/db/typesense.ts'; +import type { DBInstance } from '../src/db/index.ts'; + +const DollarToInt = (dollar: any) => { + if (dollar === null) return null; + return Math.round(dollar * 100); +} + + +// Delete and recreate the 'cards' index +export const createCardCollection = async () => { + try { + await client.collections('cards').delete(); + } catch (error) { + // Ignore error, just means collection doesn't exist + } + await client.collections().create({ + name: 'cards', + fields: [ + { name: 'id', type: 'string' }, + { name: 'cardId', type: 'int32' }, + { name: 'productId', type: 'int32' }, + { name: 'variant', type: 'string', facet: true }, + { name: 'productName', type: 'string' }, + { name: 'productLineName', type: 'string', facet: true }, + { name: 'rarityName', type: 'string', facet: true }, + { name: 'setName', type: 'string', facet: true }, + { name: 'cardType', type: 'string', facet: true }, + { name: 'energyType', type: 'string', facet: true }, + { name: 'number', type: 'string', sort: true }, + { name: 'Artist', type: 'string' }, + { name: 'sealed', type: 'bool' }, + { name: 'releaseDate', type: 'int32'}, + { name: 'content', type: 'string', token_separators: ['/'] }, + { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true } + ], + //default_sorting_field: 'productId', + }); + console.log(chalk.green('Collection "cards" created successfully.')); +} + +// Delete and recreate the 'skus' index +export const createSkuCollection = async () => { + try { + await client.collections('skus').delete(); + } catch (error) { + // Ignore error, just means collection doesn't exist + } + await client.collections().create({ + name: 'skus', + fields: [ + { name: 'id', type: 'string' }, + { name: 'condition', type: 'string' }, + { name: 'highestPrice', type: 'int32', optional: true }, + { name: 'lowestPrice', type: 'int32', optional: true }, + { name: 'marketPrice', type: 'int32', optional: true }, + ] + }); + console.log(chalk.green('Collection "skus" created successfully.')); +} + + +export const upsertCardCollection = async (db:DBInstance) => { + const pokemon = await db.query.cards.findMany({ + with: { set: true, tcgdata: true, prices: true }, + }); + await client.collections('cards').documents().import(pokemon.map(card => ({ + id: card.cardId.toString(), + cardId: card.cardId, + productId: card.productId, + variant: card.variant, + productName: card.productName, + productLineName: card.productLineName, + rarityName: card.rarityName, + setName: card.set?.setName || "", + cardType: card.cardType || "", + energyType: card.energyType || "", + number: card.number, + Artist: card.artist || "", + sealed: card.sealed, + content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '), + releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0, + sku_id: card.prices.map(price => price.skuId.toString()) + })), { action: 'upsert' }); + console.log(chalk.green('Collection "cards" indexed successfully.')); +} + +export const upsertSkuCollection = async (db:DBInstance) => { + const skus = await db.query.skus.findMany(); + await client.collections('skus').documents().import(skus.map(sku => ({ + id: sku.skuId.toString(), + condition: sku.condition, + highestPrice: DollarToInt(sku.highestPrice), + lowestPrice: DollarToInt(sku.lowestPrice), + marketPrice: DollarToInt(sku.marketPrice), + })), { action: 'upsert' }); + console.log(chalk.green('Collection "skus" indexed successfully.')); +} \ No newline at end of file diff --git a/scripts/reindex.ts b/scripts/reindex.ts index 6f552ee..613563b 100644 --- a/scripts/reindex.ts +++ b/scripts/reindex.ts @@ -1,134 +1,11 @@ -import { Client } from 'typesense'; import chalk from 'chalk'; import { db, ClosePool } from '../src/db/index.ts'; -import { client } from '../src/db/typesense.ts'; -import { release } from 'node:os'; - -const DollarToInt = (dollar: any) => { - if (dollar === null) return null; - return Math.round(dollar * 100); -} - -async function createCollection(client: Client) { - // Delete the collection if it already exists to ensure a clean slate - try { - await client.collections('cards').delete(); - await client.collections('skus').delete(); - //console.log(`Collection "cards" deleted successfully:`, response); - } catch (error) { - //console.error(`Error deleting collection "cards":`, error); - } - - // Create the collection with the specified schema - try { - await client.collections('cards').retrieve(); - console.log(chalk.yellow('Collection "cards" already exists.')); - } catch (error) { - if (error instanceof Error && error.message.includes('404')) { - await client.collections().create({ - name: 'cards', - fields: [ - { name: 'id', type: 'string' }, - { name: 'cardId', type: 'int32' }, - { name: 'productId', type: 'int32' }, - { name: 'variant', type: 'string', facet: true }, - { name: 'productName', type: 'string' }, - { name: 'productLineName', type: 'string', facet: true }, - { name: 'rarityName', type: 'string', facet: true }, - { name: 'setName', type: 'string', facet: true }, - { name: 'cardType', type: 'string', facet: true }, - { name: 'energyType', type: 'string', facet: true }, - { name: 'number', type: 'string', sort: true }, - { name: 'Artist', type: 'string' }, - { name: 'sealed', type: 'bool' }, - { name: 'releaseDate', type: 'int32'}, - { name: 'content', type: 'string', token_separators: ['/'] }, - { name: 'sku_id', type: 'string[]', optional: true, reference: 'skus.id', async_reference: true } - ], - //default_sorting_field: 'productId', - }); - console.log(chalk.green('Collection "cards" created successfully.')); - } else { - console.error(chalk.red('Error checking/creating collection:'), error); - process.exit(1); - } - } - - try { - await client.collections('skus').retrieve(); - console.log(chalk.yellow('Collection "skus" already exists.')); - } catch(error) { - if (error instanceof Error && error.message.includes('404')) { - await client.collections().create({ - name: 'skus', - fields: [ - { name: 'id', type: 'string' }, - { name: 'condition', type: 'string' }, - { name: 'highestPrice', type: 'int32', optional: true }, - { name: 'lowestPrice', type: 'int32', optional: true }, - { name: 'marketPrice', type: 'int32', optional: true }, - //{ name: 'card_id', type: 'string', reference: 'cards.id' }, - ] - }); - } - } -} +import * as Indexing from './indexing.ts'; -async function preloadSearchIndex() { - const pokemon = await db.query.cards.findMany({ - with: { set: true, tcgdata: true, prices: true }, - }); - - // Ensure the collection exists before importing documents - await createCollection(client); - - await client.collections('cards').documents().import(pokemon.map(card => ({ - id: card.cardId.toString(), - cardId: card.cardId, - productId: card.productId, - variant: card.variant, - productName: card.productName, - productLineName: card.productLineName, - rarityName: card.rarityName, - setName: card.set?.setName || "", - cardType: card.cardType || "", - energyType: card.energyType || "", - number: card.number, - Artist: card.artist || "", - sealed: card.sealed, - content: [card.productName,card.productLineName,card.set?.setName || "",card.number,card.rarityName,card.artist || ""].join(' '), - releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0, - sku_id: card.prices.map(price => price.skuId.toString()) - })), { action: 'upsert' }); - - const skus = await db.query.skus.findMany({ - with: { card: true } - }); - - await client.collections('skus').documents().import(skus.map(sku => ({ - id: sku.skuId.toString(), - condition: sku.condition, - highestPrice: DollarToInt(sku.highestPrice), - lowestPrice: DollarToInt(sku.lowestPrice), - marketPrice: DollarToInt(sku.marketPrice), - //card_id: sku.card?.cardId.toString() - }))); - - console.log(chalk.green('Search index preloaded with Pokémon cards.')); - -} - -await preloadSearchIndex().catch((error) => { - console.error(chalk.red('Error preloading search index:'), error); - for (const e of error.importResults) { - if (!e.success) { - console.error(chalk.red(`Error importing document ${e.id}:`), e.error); - } - } - process.exit(1); -}).finally(() => { - ClosePool(); - console.log(chalk.blue('Database connection closed.')); - process.exit(0); -}); +await Indexing.createCardCollection(); +await Indexing.createSkuCollection(); +await Indexing.upsertCardCollection(db); +await Indexing.upsertSkuCollection(db); +await ClosePool(); +console.log(chalk.green('Pokémon reindex complete.')); diff --git a/scripts/sync-prices.ts b/scripts/sync-prices.ts index a7d042a..2d1f0b7 100644 --- a/scripts/sync-prices.ts +++ b/scripts/sync-prices.ts @@ -3,16 +3,11 @@ import 'dotenv/config'; import chalk from 'chalk'; import { db, ClosePool } from '../src/db/index.ts'; import { sql, inArray, eq } from 'drizzle-orm'; -import { skus, processingSkus } from '../src/db/schema.ts'; -import { client } from '../src/db/typesense.ts'; +import { skus, processingSkus, priceHistory } from '../src/db/schema.ts'; import { toSnakeCase } from 'drizzle-orm/casing'; +import * as Indexing from './indexing.ts'; -const DollarToInt = (dollar: any) => { - if (dollar === null) return null; - return Math.round(dollar * 100); -} - function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } @@ -73,7 +68,7 @@ async function syncPrices() { marketPrice: sku.marketPrice, priceCount: null, }}); - await db.insert(skus).values(skuUpdates).onConflictDoUpdate({ + const skuRows = await db.insert(skus).values(skuUpdates).onConflictDoUpdate({ target: skus.skuId, set: { calculatedAt: sql.raw(`excluded.${toSnakeCase(skus.calculatedAt.name)}`), @@ -92,22 +87,10 @@ async function syncPrices() { } -async function indexPrices() { - const skus = await db.query.skus.findMany(); - - await client.collections('skus').documents().import(skus.map(sku => ({ - id: sku.skuId.toString(), - condition: sku.condition, - highestPrice: DollarToInt(sku.highestPrice), - lowestPrice: DollarToInt(sku.lowestPrice), - marketPrice: DollarToInt(sku.marketPrice), - })), { action: 'upsert' }); - -} const start = Date.now(); await syncPrices(); -await indexPrices(); +await Indexing.upsertSkuCollection(db); await ClosePool(); const end = Date.now(); const duration = (end - start) / 1000; diff --git a/src/db/index.ts b/src/db/index.ts index e7512df..928e2e2 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -17,6 +17,7 @@ pool.on('error', (err) => { }); export const db = drizzle({ client: pool, relations: relations, casing: 'snake_case' }); +export type DBInstance = typeof db; export const ClosePool = () => { pool.end();