preload import working
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -22,3 +22,6 @@ pnpm-debug.log*
|
||||
|
||||
# jetbrains setting folder
|
||||
.idea/
|
||||
|
||||
# imges from tcgplayer
|
||||
public/cards/*
|
||||
|
||||
11
drizzle.config.ts
Normal file
11
drizzle.config.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import 'dotenv/config'; // Import dotenv to load environment variables
|
||||
import { defineConfig } from 'drizzle-kit';
|
||||
|
||||
export default defineConfig({
|
||||
out: './drizzle', // Directory for migration files
|
||||
schema: './src/db/schema.ts', // Path to your schema file
|
||||
dialect: 'mysql', // Specify the database dialect
|
||||
dbCredentials: {
|
||||
url: process.env.DATABASE_URL!, // Use the URL from your .env file
|
||||
},
|
||||
});
|
||||
1
package-lock.json
generated
1
package-lock.json
generated
@@ -9,6 +9,7 @@
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"astro": "^5.17.1",
|
||||
"chalk": "^5.6.2",
|
||||
"dotenv": "^17.2.4",
|
||||
"drizzle-orm": "^0.45.1",
|
||||
"mysql2": "^3.16.3"
|
||||
|
||||
@@ -6,10 +6,12 @@
|
||||
"dev": "astro dev",
|
||||
"build": "astro build",
|
||||
"preview": "astro preview",
|
||||
"astro": "astro"
|
||||
"astro": "astro",
|
||||
"add-user": "ts-node scripts/add-user.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"astro": "^5.17.1",
|
||||
"chalk": "^5.6.2",
|
||||
"dotenv": "^17.2.4",
|
||||
"drizzle-orm": "^0.45.1",
|
||||
"mysql2": "^3.16.3"
|
||||
|
||||
308
scripts/preload-tcgplayer.ts
Normal file
308
scripts/preload-tcgplayer.ts
Normal file
@@ -0,0 +1,308 @@
|
||||
import 'dotenv/config';
|
||||
import { drizzle } from 'drizzle-orm/mysql2';
|
||||
import mysql from 'mysql2/promise';
|
||||
import * as schema from '../src/db/schema.ts';
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { eq } from 'drizzle-orm';
|
||||
import chalk from 'chalk';
|
||||
//import util from 'util';
|
||||
|
||||
|
||||
async function syncTcgplayer() {
|
||||
|
||||
// const productLines = [
|
||||
// { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
|
||||
// { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }
|
||||
// ];
|
||||
|
||||
const productLines = [
|
||||
{ name: "pokemon-japan", cardType: ["Dragon", "Colorless", "Energy"] }
|
||||
];
|
||||
|
||||
for (const productLine of productLines) {
|
||||
for (const [key, values] of Object.entries(productLine)) {
|
||||
if (key === "name") continue;
|
||||
for (const value of values) {
|
||||
console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
|
||||
await syncProductLineEnergyType(productLine.name, key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function fileExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) {
|
||||
let start = 0;
|
||||
let size = 50;
|
||||
let total = 1000000;
|
||||
|
||||
while (start < total) {
|
||||
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
||||
|
||||
|
||||
let d = {
|
||||
"algorithm":"sales_dismax",
|
||||
"from":start,
|
||||
"size":size,
|
||||
"filters":{
|
||||
"term":{"productLineName":[productLine]},
|
||||
"range":{},
|
||||
"match":{}
|
||||
},
|
||||
"listingSearch":{
|
||||
"context":{"cart":{}},
|
||||
"filters":{"term":{
|
||||
"sellerStatus":"Live",
|
||||
"channelId":0
|
||||
},
|
||||
"range":{
|
||||
"quantity":{"gte":1}
|
||||
},
|
||||
"exclude":{"channelExclusion":0}
|
||||
}
|
||||
},
|
||||
"context":{
|
||||
"cart":{},
|
||||
"shippingCountry":"US",
|
||||
"userProfile":{}
|
||||
},
|
||||
"settings":{
|
||||
"useFuzzySearch":false,
|
||||
"didYouMean":{}
|
||||
},
|
||||
"sort":{}
|
||||
};
|
||||
d.filters.term[field] = [fieldValue];
|
||||
|
||||
//console.log(util.inspect(d, { depth: null }));
|
||||
//process.exit(1);
|
||||
|
||||
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(d),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error('Error notifying sync completion:', response.statusText);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
total = data.results[0].totalResults;
|
||||
|
||||
//console.log(data);
|
||||
|
||||
const poolConnection = mysql.createPool({
|
||||
uri: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
const db = drizzle(poolConnection, { schema, mode: 'default' });
|
||||
|
||||
for (const item of data.results[0].results) {
|
||||
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
|
||||
|
||||
await db.insert(schema.cards).values({
|
||||
productId: item.productId,
|
||||
productName: item.productName,
|
||||
rarityName: item.rarityName,
|
||||
productLineName: item.productLineName,
|
||||
productLineUrlName: item.productLineUrlName,
|
||||
productStatusId: item.productStatusId,
|
||||
productTypeId: item.productTypeId,
|
||||
productUrlName: item.productUrlName,
|
||||
setId: item.setId,
|
||||
shippingCategoryId: item.shippingCategoryId,
|
||||
sealed: item.sealed,
|
||||
sellerListable: item.sellerListable,
|
||||
foilOnly: item.foilOnly,
|
||||
attack1: item.customAttributes.attack1 || null,
|
||||
attack2: item.customAttributes.attack2 || null,
|
||||
attack3: item.customAttributes.attack3 || null,
|
||||
attack4: item.customAttributes.attack4 || null,
|
||||
cardType: item.customAttributes.cardType?.[0] || null,
|
||||
cardTypeB: item.customAttributes.cardTypeB || null,
|
||||
energyType: item.customAttributes.energyType?.[0] || null,
|
||||
flavorText: item.customAttributes.flavorText || null,
|
||||
hp: item.customAttributes.hp || 0,
|
||||
number: item.customAttributes.number || '',
|
||||
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
||||
resistance: item.customAttributes.resistance || null,
|
||||
retreatCost: item.customAttributes.retreatCost || null,
|
||||
stage: item.customAttributes.stage || null,
|
||||
weakness: item.customAttributes.weakness || null,
|
||||
lowestPrice: item.lowestPrice,
|
||||
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
||||
marketPrice: item.marketPrice,
|
||||
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
||||
medianPrice: item.medianPrice,
|
||||
totalListings: item.totalListings,
|
||||
}).onDuplicateKeyUpdate({
|
||||
set: {
|
||||
productName: item.productName,
|
||||
rarityName: item.rarityName,
|
||||
productLineName: item.productLineName,
|
||||
productLineUrlName: item.productLineUrlName,
|
||||
productStatusId: item.productStatusId,
|
||||
productTypeId: item.productTypeId,
|
||||
productUrlName: item.productUrlName,
|
||||
setId: item.setId,
|
||||
shippingCategoryId: item.shippingCategoryId,
|
||||
sealed: item.sealed,
|
||||
sellerListable: item.sellerListable,
|
||||
foilOnly: item.foilOnly,
|
||||
attack1: item.customAttributes.attack1 || null,
|
||||
attack2: item.customAttributes.attack2 || null,
|
||||
attack3: item.customAttributes.attack3 || null,
|
||||
attack4: item.customAttributes.attack4 || null,
|
||||
cardType: item.customAttributes.cardType?.[0] || null,
|
||||
cardTypeB: item.customAttributes.cardTypeB || null,
|
||||
energyType: item.customAttributes.energyType?.[0] || null,
|
||||
flavorText: item.customAttributes.flavorText || null,
|
||||
hp: item.customAttributes.hp || 0,
|
||||
number: item.customAttributes.number || '',
|
||||
releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
|
||||
resistance: item.customAttributes.resistance || null,
|
||||
retreatCost: item.customAttributes.retreatCost || null,
|
||||
stage: item.customAttributes.stage || null,
|
||||
weakness: item.customAttributes.weakness || null,
|
||||
lowestPrice: item.lowestPrice,
|
||||
lowestPriceWithShipping: item.lowestPriceWithShipping,
|
||||
marketPrice: item.marketPrice,
|
||||
maxFulfillableQuantity: item.maxFulfillableQuantity,
|
||||
medianPrice: item.medianPrice,
|
||||
totalListings: item.totalListings,
|
||||
},
|
||||
});
|
||||
|
||||
// before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls.
|
||||
const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId));
|
||||
const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000);
|
||||
if (hasRecentSku) {
|
||||
console.log(chalk.blue(' Skipping details and pricing fetch since we have recent SKU data'));
|
||||
await sleep(100);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get product detail
|
||||
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
if (!detailResponse.ok) {
|
||||
console.error('Error fetching product details:', detailResponse.statusText);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const detailData = await detailResponse.json();
|
||||
|
||||
await db.insert(schema.sets).values({
|
||||
setId: detailData.setId,
|
||||
setCode: detailData.setCode,
|
||||
setName: detailData.setName,
|
||||
setUrlName: detailData.setUrlName,
|
||||
}).onDuplicateKeyUpdate({
|
||||
set: {
|
||||
setCode: detailData.setCode,
|
||||
setName: detailData.setName,
|
||||
setUrlName: detailData.setUrlName,
|
||||
},
|
||||
});
|
||||
|
||||
// skus are...
|
||||
const skuArray = detailData.skus.map((sku: any) => sku.sku);
|
||||
//console.log(detailData.skus);
|
||||
//console.log(skuArray);
|
||||
// get pricing for skus
|
||||
const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ skuIds: skuArray }),
|
||||
});
|
||||
|
||||
if (!skuResponse.ok) {
|
||||
console.error('Error fetching SKU pricing:', skuResponse.statusText);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const skuData = await skuResponse.json();
|
||||
|
||||
let skuMap = new Map();
|
||||
for (const skuItem of skuData) {
|
||||
skuMap.set(skuItem.skuId, skuItem);
|
||||
}
|
||||
|
||||
for (const skuItem of detailData.skus) {
|
||||
const pricing = skuMap.get(skuItem.sku);
|
||||
//console.log(pricing);
|
||||
|
||||
await db.insert(schema.skus).values({
|
||||
skuId: skuItem.sku,
|
||||
productId: detailData.productId,
|
||||
condition: skuItem.condition,
|
||||
language: skuItem.language,
|
||||
variant: skuItem.variant,
|
||||
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
|
||||
highestPrice: pricing?.highestPrice || null,
|
||||
lowestPrice: pricing?.lowestPrice || null,
|
||||
marketPrice: pricing?.marketPrice || null,
|
||||
priceCount: pricing?.priceCount || 0,
|
||||
}).onDuplicateKeyUpdate({
|
||||
set: {
|
||||
condition: skuItem.condition,
|
||||
language: skuItem.language,
|
||||
variant: skuItem.variant,
|
||||
calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
|
||||
highestPrice: pricing?.highestPrice || null,
|
||||
lowestPrice: pricing?.lowestPrice || null,
|
||||
marketPrice: pricing?.marketPrice || null,
|
||||
priceCount: pricing?.priceCount || 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// get image if it doesn't already exist
|
||||
const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
|
||||
if (!await fileExists(imagePath)) {
|
||||
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
|
||||
if (imageResponse.ok) {
|
||||
const buffer = await imageResponse.arrayBuffer();
|
||||
await fs.writeFile(imagePath, Buffer.from(buffer));
|
||||
} else {
|
||||
console.error('Error fetching product image:', imageResponse.statusText);
|
||||
}
|
||||
}
|
||||
|
||||
// be nice to the API and not send too many requests in a short time
|
||||
await sleep(100);
|
||||
|
||||
}
|
||||
|
||||
|
||||
await poolConnection.end();
|
||||
|
||||
start += size;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
syncTcgplayer();
|
||||
@@ -1,9 +1,63 @@
|
||||
// src/db/schema.ts
|
||||
import { mysqlTable, serial, varchar, int } from 'drizzle-orm/mysql-core';
|
||||
import { mysqlTable, varchar, int, boolean, decimal, datetime, index } from 'drizzle-orm/mysql-core';
|
||||
|
||||
export const users = mysqlTable('users', {
|
||||
id: serial('id').notNull().primaryKey(),
|
||||
name: varchar('name', { length: 255 }).notNull(),
|
||||
age: int('age').notNull(),
|
||||
email: varchar('email', { length: 255 }).notNull(),
|
||||
export const cards = mysqlTable('cards', {
|
||||
productId: int().notNull().primaryKey(),
|
||||
productName: varchar({ length: 255 }).notNull(),
|
||||
productLineName: varchar({ length: 255 }).notNull().default(''),
|
||||
productLineUrlName: varchar({ length: 255 }).notNull().default(''),
|
||||
productStatusId: int().notNull().default(0),
|
||||
productTypeId: int().notNull().default(0),
|
||||
productUrlName: varchar({ length: 255 }).notNull().default(''),
|
||||
rarityName: varchar({ length: 100 }).notNull().default(''),
|
||||
score: decimal({ precision: 10, scale: 2 }).notNull().default('0'),
|
||||
sealed: boolean().notNull().default(false),
|
||||
sellerListable: boolean().notNull().default(false),
|
||||
setId: int().notNull().default(0),
|
||||
shippingCategoryId: int().notNull().default(0),
|
||||
duplicate: boolean().notNull().default(false),
|
||||
foilOnly: boolean().notNull().default(false),
|
||||
attack1: varchar({ length: 1024 }),
|
||||
attack2: varchar({ length: 1024 }),
|
||||
attack3: varchar({ length: 1024 }),
|
||||
attack4: varchar({ length: 1024 }),
|
||||
cardType: varchar({ length: 100 }),
|
||||
cardTypeB: varchar({ length: 100 }),
|
||||
energyType: varchar({ length: 100 }),
|
||||
flavorText: varchar({ length: 1000 }),
|
||||
hp: int().notNull().default(0),
|
||||
number: varchar({ length: 50 }).notNull().default(''),
|
||||
releaseDate: datetime(),
|
||||
resistance: varchar({ length: 100 }),
|
||||
retreatCost: varchar({ length: 100 }),
|
||||
stage: varchar({ length: 100 }),
|
||||
weakness: varchar({ length: 100 }),
|
||||
lowestPrice: decimal({ precision: 10, scale: 2 }).notNull().default('0'),
|
||||
lowestPriceWithShipping: decimal({ precision: 10, scale: 2 }).notNull().default('0'),
|
||||
marketPrice: decimal({ precision: 10, scale: 2 }).notNull().default('0'),
|
||||
maxFulfillableQuantity: int().notNull().default(0),
|
||||
medianPrice: decimal({ precision: 10, scale: 2 }).notNull().default('0'),
|
||||
totalListings: int().notNull().default(0),
|
||||
});
|
||||
|
||||
export const sets = mysqlTable('sets', {
|
||||
setId: int().notNull().primaryKey(),
|
||||
setCode: varchar({ length: 100 }).notNull(),
|
||||
setName: varchar({ length: 255 }).notNull(),
|
||||
setUrlName: varchar({ length: 255 }).notNull(),
|
||||
});
|
||||
|
||||
export const skus = mysqlTable('skus', {
|
||||
skuId: int().notNull().primaryKey(),
|
||||
productId: int().notNull(),
|
||||
condition: varchar({ length: 255 }).notNull(),
|
||||
language: varchar({ length: 100 }).notNull(),
|
||||
variant: varchar({ length: 100 }).notNull(),
|
||||
calculatedAt: datetime(),
|
||||
highestPrice: decimal({ precision: 10, scale: 2 }),
|
||||
lowestPrice: decimal({ precision: 10, scale: 2 }),
|
||||
marketPrice: decimal({ precision: 10, scale: 2 }),
|
||||
priceCount: int(),
|
||||
},(table) => ({
|
||||
productIdIdx: index('productIdIdx').on(table.productId),
|
||||
}));
|
||||
@@ -1,14 +0,0 @@
|
||||
---
|
||||
// src/pages/users.astro
|
||||
import { db } from '../db/index';
|
||||
import { users } from '../db/schema';
|
||||
|
||||
const allUsers = await db.select().from(users);
|
||||
---
|
||||
|
||||
<h1>User List</h1>
|
||||
<ul>
|
||||
{allUsers.map((user) => (
|
||||
<li>{user.name} ({user.email})</li>
|
||||
))}
|
||||
</ul>
|
||||
Reference in New Issue
Block a user