diff --git a/.gitignore b/.gitignore index 016b59e..cb7f03c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ pnpm-debug.log* # jetbrains setting folder .idea/ + +# imges from tcgplayer +public/cards/* diff --git a/drizzle.config.ts b/drizzle.config.ts new file mode 100644 index 0000000..dfcd666 --- /dev/null +++ b/drizzle.config.ts @@ -0,0 +1,11 @@ +import 'dotenv/config'; // Import dotenv to load environment variables +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + out: './drizzle', // Directory for migration files + schema: './src/db/schema.ts', // Path to your schema file + dialect: 'mysql', // Specify the database dialect + dbCredentials: { + url: process.env.DATABASE_URL!, // Use the URL from your .env file + }, +}); diff --git a/package-lock.json b/package-lock.json index c3a0c80..6c4b47c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "astro": "^5.17.1", + "chalk": "^5.6.2", "dotenv": "^17.2.4", "drizzle-orm": "^0.45.1", "mysql2": "^3.16.3" diff --git a/package.json b/package.json index 28e4ce7..0524273 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,12 @@ "dev": "astro dev", "build": "astro build", "preview": "astro preview", - "astro": "astro" + "astro": "astro", + "add-user": "ts-node scripts/add-user.ts" }, "dependencies": { "astro": "^5.17.1", + "chalk": "^5.6.2", "dotenv": "^17.2.4", "drizzle-orm": "^0.45.1", "mysql2": "^3.16.3" diff --git a/scripts/preload-tcgplayer.ts b/scripts/preload-tcgplayer.ts new file mode 100644 index 0000000..88bd0eb --- /dev/null +++ b/scripts/preload-tcgplayer.ts @@ -0,0 +1,308 @@ +import 'dotenv/config'; +import { drizzle } from 'drizzle-orm/mysql2'; +import mysql from 'mysql2/promise'; +import * as schema from '../src/db/schema.ts'; +import fs from "node:fs/promises"; +import path from "node:path"; +import { eq } from 'drizzle-orm'; +import chalk from 'chalk'; +//import util from 'util'; + + +async function syncTcgplayer() { + + // const productLines = [ + // { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }, + // { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] } + // ]; + + const productLines = [ + { name: "pokemon-japan", cardType: ["Dragon", "Colorless", "Energy"] } + ]; + + for (const productLine of productLines) { + for (const [key, values] of Object.entries(productLine)) { + if (key === "name") continue; + for (const value of values) { + console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`); + await syncProductLineEnergyType(productLine.name, key, value); + } + } + } + + console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!')); +} + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function fileExists(path: string): Promise { + try { + await fs.access(path); + return true; + } catch { + return false; + } +} + +async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) { + let start = 0; + let size = 50; + let total = 1000000; + + while (start < total) { + console.log(` Fetching items ${start} to ${start + size} of ${total}...`); + + + let d = { + "algorithm":"sales_dismax", + "from":start, + "size":size, + "filters":{ + "term":{"productLineName":[productLine]}, + "range":{}, + "match":{} + }, + "listingSearch":{ + "context":{"cart":{}}, + "filters":{"term":{ + "sellerStatus":"Live", + "channelId":0 + }, + "range":{ + "quantity":{"gte":1} + }, + "exclude":{"channelExclusion":0} + } + }, + "context":{ + "cart":{}, + "shippingCountry":"US", + "userProfile":{} + }, + "settings":{ + "useFuzzySearch":false, + "didYouMean":{} + }, + "sort":{} + }; + d.filters.term[field] = [fieldValue]; + + //console.log(util.inspect(d, { depth: null })); + //process.exit(1); + + const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(d), + }); + + if (!response.ok) { + console.error('Error notifying sync completion:', response.statusText); + process.exit(1); + } + + const data = await response.json(); + total = data.results[0].totalResults; + + //console.log(data); + + const poolConnection = mysql.createPool({ + uri: process.env.DATABASE_URL, + }); + + const db = drizzle(poolConnection, { schema, mode: 'default' }); + + for (const item of data.results[0].results) { + console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`)); + + await db.insert(schema.cards).values({ + productId: item.productId, + productName: item.productName, + rarityName: item.rarityName, + productLineName: item.productLineName, + productLineUrlName: item.productLineUrlName, + productStatusId: item.productStatusId, + productTypeId: item.productTypeId, + productUrlName: item.productUrlName, + setId: item.setId, + shippingCategoryId: item.shippingCategoryId, + sealed: item.sealed, + sellerListable: item.sellerListable, + foilOnly: item.foilOnly, + attack1: item.customAttributes.attack1 || null, + attack2: item.customAttributes.attack2 || null, + attack3: item.customAttributes.attack3 || null, + attack4: item.customAttributes.attack4 || null, + cardType: item.customAttributes.cardType?.[0] || null, + cardTypeB: item.customAttributes.cardTypeB || null, + energyType: item.customAttributes.energyType?.[0] || null, + flavorText: item.customAttributes.flavorText || null, + hp: item.customAttributes.hp || 0, + number: item.customAttributes.number || '', + releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, + resistance: item.customAttributes.resistance || null, + retreatCost: item.customAttributes.retreatCost || null, + stage: item.customAttributes.stage || null, + weakness: item.customAttributes.weakness || null, + lowestPrice: item.lowestPrice, + lowestPriceWithShipping: item.lowestPriceWithShipping, + marketPrice: item.marketPrice, + maxFulfillableQuantity: item.maxFulfillableQuantity, + medianPrice: item.medianPrice, + totalListings: item.totalListings, + }).onDuplicateKeyUpdate({ + set: { + productName: item.productName, + rarityName: item.rarityName, + productLineName: item.productLineName, + productLineUrlName: item.productLineUrlName, + productStatusId: item.productStatusId, + productTypeId: item.productTypeId, + productUrlName: item.productUrlName, + setId: item.setId, + shippingCategoryId: item.shippingCategoryId, + sealed: item.sealed, + sellerListable: item.sellerListable, + foilOnly: item.foilOnly, + attack1: item.customAttributes.attack1 || null, + attack2: item.customAttributes.attack2 || null, + attack3: item.customAttributes.attack3 || null, + attack4: item.customAttributes.attack4 || null, + cardType: item.customAttributes.cardType?.[0] || null, + cardTypeB: item.customAttributes.cardTypeB || null, + energyType: item.customAttributes.energyType?.[0] || null, + flavorText: item.customAttributes.flavorText || null, + hp: item.customAttributes.hp || 0, + number: item.customAttributes.number || '', + releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null, + resistance: item.customAttributes.resistance || null, + retreatCost: item.customAttributes.retreatCost || null, + stage: item.customAttributes.stage || null, + weakness: item.customAttributes.weakness || null, + lowestPrice: item.lowestPrice, + lowestPriceWithShipping: item.lowestPriceWithShipping, + marketPrice: item.marketPrice, + maxFulfillableQuantity: item.maxFulfillableQuantity, + medianPrice: item.medianPrice, + totalListings: item.totalListings, + }, + }); + + // before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls. + const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId)); + const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000); + if (hasRecentSku) { + console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data')); + await sleep(100); + continue; + } + + // Get product detail + const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, { + method: 'GET', + }); + + if (!detailResponse.ok) { + console.error('Error fetching product details:', detailResponse.statusText); + process.exit(1); + } + + const detailData = await detailResponse.json(); + + await db.insert(schema.sets).values({ + setId: detailData.setId, + setCode: detailData.setCode, + setName: detailData.setName, + setUrlName: detailData.setUrlName, + }).onDuplicateKeyUpdate({ + set: { + setCode: detailData.setCode, + setName: detailData.setName, + setUrlName: detailData.setUrlName, + }, + }); + + // skus are... + const skuArray = detailData.skus.map((sku: any) => sku.sku); + //console.log(detailData.skus); + //console.log(skuArray); + // get pricing for skus + const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ skuIds: skuArray }), + }); + + if (!skuResponse.ok) { + console.error('Error fetching SKU pricing:', skuResponse.statusText); + process.exit(1); + } + + const skuData = await skuResponse.json(); + + let skuMap = new Map(); + for (const skuItem of skuData) { + skuMap.set(skuItem.skuId, skuItem); + } + + for (const skuItem of detailData.skus) { + const pricing = skuMap.get(skuItem.sku); + //console.log(pricing); + + await db.insert(schema.skus).values({ + skuId: skuItem.sku, + productId: detailData.productId, + condition: skuItem.condition, + language: skuItem.language, + variant: skuItem.variant, + calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null, + highestPrice: pricing?.highestPrice || null, + lowestPrice: pricing?.lowestPrice || null, + marketPrice: pricing?.marketPrice || null, + priceCount: pricing?.priceCount || 0, + }).onDuplicateKeyUpdate({ + set: { + condition: skuItem.condition, + language: skuItem.language, + variant: skuItem.variant, + calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null, + highestPrice: pricing?.highestPrice || null, + lowestPrice: pricing?.lowestPrice || null, + marketPrice: pricing?.marketPrice || null, + priceCount: pricing?.priceCount || 0, + }, + }); + } + + // get image if it doesn't already exist + const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`); + if (!await fileExists(imagePath)) { + const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`); + if (imageResponse.ok) { + const buffer = await imageResponse.arrayBuffer(); + await fs.writeFile(imagePath, Buffer.from(buffer)); + } else { + console.error('Error fetching product image:', imageResponse.statusText); + } + } + + // be nice to the API and not send too many requests in a short time + await sleep(100); + + } + + + await poolConnection.end(); + + start += size; + } +} + + +syncTcgplayer(); diff --git a/src/db/schema.ts b/src/db/schema.ts index aa9c939..47e9254 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,9 +1,63 @@ // src/db/schema.ts -import { mysqlTable, serial, varchar, int } from 'drizzle-orm/mysql-core'; +import { mysqlTable, varchar, int, boolean, decimal, datetime, index } from 'drizzle-orm/mysql-core'; -export const users = mysqlTable('users', { - id: serial('id').notNull().primaryKey(), - name: varchar('name', { length: 255 }).notNull(), - age: int('age').notNull(), - email: varchar('email', { length: 255 }).notNull(), +export const cards = mysqlTable('cards', { + productId: int().notNull().primaryKey(), + productName: varchar({ length: 255 }).notNull(), + productLineName: varchar({ length: 255 }).notNull().default(''), + productLineUrlName: varchar({ length: 255 }).notNull().default(''), + productStatusId: int().notNull().default(0), + productTypeId: int().notNull().default(0), + productUrlName: varchar({ length: 255 }).notNull().default(''), + rarityName: varchar({ length: 100 }).notNull().default(''), + score: decimal({ precision: 10, scale: 2 }).notNull().default('0'), + sealed: boolean().notNull().default(false), + sellerListable: boolean().notNull().default(false), + setId: int().notNull().default(0), + shippingCategoryId: int().notNull().default(0), + duplicate: boolean().notNull().default(false), + foilOnly: boolean().notNull().default(false), + attack1: varchar({ length: 1024 }), + attack2: varchar({ length: 1024 }), + attack3: varchar({ length: 1024 }), + attack4: varchar({ length: 1024 }), + cardType: varchar({ length: 100 }), + cardTypeB: varchar({ length: 100 }), + energyType: varchar({ length: 100 }), + flavorText: varchar({ length: 1000 }), + hp: int().notNull().default(0), + number: varchar({ length: 50 }).notNull().default(''), + releaseDate: datetime(), + resistance: varchar({ length: 100 }), + retreatCost: varchar({ length: 100 }), + stage: varchar({ length: 100 }), + weakness: varchar({ length: 100 }), + lowestPrice: decimal({ precision: 10, scale: 2 }).notNull().default('0'), + lowestPriceWithShipping: decimal({ precision: 10, scale: 2 }).notNull().default('0'), + marketPrice: decimal({ precision: 10, scale: 2 }).notNull().default('0'), + maxFulfillableQuantity: int().notNull().default(0), + medianPrice: decimal({ precision: 10, scale: 2 }).notNull().default('0'), + totalListings: int().notNull().default(0), }); + +export const sets = mysqlTable('sets', { + setId: int().notNull().primaryKey(), + setCode: varchar({ length: 100 }).notNull(), + setName: varchar({ length: 255 }).notNull(), + setUrlName: varchar({ length: 255 }).notNull(), +}); + +export const skus = mysqlTable('skus', { + skuId: int().notNull().primaryKey(), + productId: int().notNull(), + condition: varchar({ length: 255 }).notNull(), + language: varchar({ length: 100 }).notNull(), + variant: varchar({ length: 100 }).notNull(), + calculatedAt: datetime(), + highestPrice: decimal({ precision: 10, scale: 2 }), + lowestPrice: decimal({ precision: 10, scale: 2 }), + marketPrice: decimal({ precision: 10, scale: 2 }), + priceCount: int(), +},(table) => ({ + productIdIdx: index('productIdIdx').on(table.productId), +})); \ No newline at end of file diff --git a/src/pages/test.astro b/src/pages/test.astro deleted file mode 100644 index 06b9668..0000000 --- a/src/pages/test.astro +++ /dev/null @@ -1,14 +0,0 @@ ---- -// src/pages/users.astro -import { db } from '../db/index'; -import { users } from '../db/schema'; - -const allUsers = await db.select().from(users); ---- - -

User List

-