[feat] tcg player import added to admin page
This commit is contained in:
@@ -208,7 +208,7 @@ export const upsertInventoryCollection = async (db:DBInstance, log: Logger = def
|
||||
|
||||
|
||||
|
||||
export const UpdateVariants = async (db:DBInstance) => {
|
||||
export const UpdateVariants = async (db:DBInstance, log: Logger = (m) => console.log(m)) => {
|
||||
const updates = await db.execute(sql`update cards as c
|
||||
set
|
||||
product_name = a.product_name, product_line_name = a.product_line_name, product_url_name = a.product_url_name, rarity_name = a.rarity_name,
|
||||
@@ -232,7 +232,7 @@ where c.product_id = a.product_id and c.variant = a.variant and
|
||||
c.energy_type is distinct from a.energy_type or c."number" is distinct from a."number" or c.artist is distinct from a.artist
|
||||
)
|
||||
`);
|
||||
console.log(`Updated ${updates.rowCount} rows in cards table`);
|
||||
log(`Updated ${updates.rowCount} rows in cards table`);
|
||||
|
||||
const inserts = await db.execute(sql`insert into cards (product_id, variant, product_name, product_line_name, product_url_name, rarity_name, sealed, set_id, card_type, energy_type, "number", artist)
|
||||
select t.product_id, b.variant,
|
||||
@@ -245,9 +245,9 @@ join (select distinct product_id, variant from skus) b on t.product_id = b.produ
|
||||
left join tcg_overrides o on t.product_id = o.product_id
|
||||
where not exists (select 1 from cards where product_id=t.product_id and variant=b.variant)
|
||||
`);
|
||||
console.log(`Inserted ${inserts.rowCount} rows into cards table`);
|
||||
log(`Inserted ${inserts.rowCount} rows into cards table`);
|
||||
|
||||
const skuUpdates = await db.execute(sql`update skus s set card_id = c.card_id from cards c where s.product_id = c.product_id and s.variant = c.variant and s.card_id is distinct from c.card_id`);
|
||||
console.log(`Updated ${skuUpdates.rowCount} rows in skus table`);
|
||||
log(`Updated ${skuUpdates.rowCount} rows in skus table`);
|
||||
|
||||
}
|
||||
|
||||
@@ -1,106 +1,68 @@
|
||||
import 'dotenv/config';
|
||||
import * as schema from '../src/db/schema.ts';
|
||||
import { db, ClosePool } from '../src/db/index.ts';
|
||||
import { db, ClosePool, type DBInstance } from '../src/db/index.ts';
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import chalk from 'chalk';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import * as helper from './pokemon-helper.ts';
|
||||
//import util from 'util';
|
||||
|
||||
export type Logger = (msg: string) => void;
|
||||
const consoleLogger: Logger = (m) => console.log(m);
|
||||
|
||||
export type RunImportOptions = {
|
||||
sets?: string[];
|
||||
log?: Logger;
|
||||
runUpdateVariants?: boolean;
|
||||
runCardUpsert?: boolean;
|
||||
};
|
||||
|
||||
|
||||
async function syncTcgplayer(cardSets:string[] = []) {
|
||||
|
||||
const productLines = [ "pokemon", "pokemon-japan" ];
|
||||
|
||||
// work from the available sets within the product line
|
||||
for (const productLine of productLines) {
|
||||
const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};
|
||||
|
||||
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json',},
|
||||
body: JSON.stringify(d),
|
||||
});
|
||||
if (!response.ok) {
|
||||
console.error('Error notifying sync completion:', response.statusText);
|
||||
process.exit(1);
|
||||
}
|
||||
const data = await response.json();
|
||||
|
||||
const setNames = data.results[0].aggregations.setName;
|
||||
for (const setName of setNames) {
|
||||
let processSet = true;
|
||||
if (cardSets.length > 0) {
|
||||
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
|
||||
}
|
||||
if (processSet) {
|
||||
console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
|
||||
await syncProductLine(productLine, "setName", setName.urlValue);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
|
||||
}
|
||||
|
||||
|
||||
async function syncProductLine(productLine: string, field: string, fieldValue: string) {
|
||||
const syncProductLine = async (
|
||||
database: DBInstance,
|
||||
productLine: string,
|
||||
field: string,
|
||||
fieldValue: string,
|
||||
allProductIds: Set<number>,
|
||||
log: Logger,
|
||||
) => {
|
||||
let start = 0;
|
||||
let size = 50;
|
||||
const size = 50;
|
||||
let total = 1000000;
|
||||
|
||||
while (start < total) {
|
||||
console.log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
||||
log(` Fetching items ${start} to ${start + size} of ${total}...`);
|
||||
|
||||
const d = {
|
||||
"algorithm":"sales_dismax",
|
||||
"from":start,
|
||||
"size":size,
|
||||
"filters":{
|
||||
"term":{"productLineName":[productLine], [field]:[fieldValue]} ,
|
||||
"range":{},
|
||||
"match":{}
|
||||
},
|
||||
"listingSearch":{
|
||||
"context":{"cart":{}},
|
||||
"filters":{"term":{
|
||||
"sellerStatus":"Live",
|
||||
"channelId":0
|
||||
},
|
||||
"range":{
|
||||
"quantity":{"gte":1}
|
||||
},
|
||||
"exclude":{"channelExclusion":0}
|
||||
}
|
||||
},
|
||||
"context":{
|
||||
"cart":{},
|
||||
"shippingCountry":"US",
|
||||
"userProfile":{}
|
||||
},
|
||||
"settings":{
|
||||
"useFuzzySearch":false,
|
||||
"didYouMean":{}
|
||||
},
|
||||
"sort":{}
|
||||
};
|
||||
|
||||
//console.log(util.inspect(d, { depth: null }));
|
||||
//process.exit(1);
|
||||
"algorithm": "sales_dismax",
|
||||
"from": start,
|
||||
"size": size,
|
||||
"filters": {
|
||||
"term": { "productLineName": [productLine], [field]: [fieldValue] },
|
||||
"range": {},
|
||||
"match": {},
|
||||
},
|
||||
"listingSearch": {
|
||||
"context": { "cart": {} },
|
||||
"filters": {
|
||||
"term": { "sellerStatus": "Live", "channelId": 0 },
|
||||
"range": { "quantity": { "gte": 1 } },
|
||||
"exclude": { "channelExclusion": 0 },
|
||||
},
|
||||
},
|
||||
"context": { "cart": {}, "shippingCountry": "US", "userProfile": {} },
|
||||
"settings": { "useFuzzySearch": false, "didYouMean": {} },
|
||||
"sort": {},
|
||||
};
|
||||
|
||||
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(d),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error('Error notifying sync completion:', response.statusText);
|
||||
process.exit(1);
|
||||
throw new Error(`TCGPlayer search request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
@@ -108,28 +70,24 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
|
||||
for (const item of data.results[0].results) {
|
||||
|
||||
// Check if productId already exists and skip if it does (to avoid hitting the API too much)
|
||||
if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));
|
||||
log(` - ${item.productName} (ID: ${item.productId})`);
|
||||
|
||||
// Get product detail
|
||||
const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
|
||||
method: 'GET',
|
||||
});
|
||||
if (!detailResponse.ok) {
|
||||
console.error('Error fetching product details:', detailResponse.statusText);
|
||||
process.exit(1);
|
||||
throw new Error(`Error fetching product details for ${item.productId}: ${detailResponse.statusText}`);
|
||||
}
|
||||
const detailData = await detailResponse.json();
|
||||
|
||||
|
||||
await db.insert(schema.tcgcards).values({
|
||||
await database.insert(schema.tcgcards).values({
|
||||
productId: item.productId,
|
||||
productName: detailData.productName,
|
||||
//productName: cleanProductName(item.productName),
|
||||
rarityName: item.rarityName,
|
||||
productLineName: detailData.productLineName,
|
||||
productLineUrlName: detailData.productLineUrlName,
|
||||
@@ -167,7 +125,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
target: schema.tcgcards.productId,
|
||||
set: {
|
||||
productName: detailData.productName,
|
||||
//productName: cleanProductName(item.productName),
|
||||
rarityName: item.rarityName,
|
||||
productLineName: detailData.productLineName,
|
||||
productLineUrlName: detailData.productLineUrlName,
|
||||
@@ -204,11 +161,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
},
|
||||
});
|
||||
|
||||
// console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
|
||||
// console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
|
||||
// console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
|
||||
// set is...
|
||||
await db.insert(schema.sets).values({
|
||||
await database.insert(schema.sets).values({
|
||||
setId: detailData.setId,
|
||||
setCode: detailData.setCode,
|
||||
setName: detailData.setName,
|
||||
@@ -222,10 +175,8 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
},
|
||||
});
|
||||
|
||||
// skus are...
|
||||
for (const skuItem of detailData.skus) {
|
||||
|
||||
await db.insert(schema.skus).values({
|
||||
await database.insert(schema.skus).values({
|
||||
skuId: skuItem.sku,
|
||||
productId: detailData.productId,
|
||||
condition: skuItem.condition,
|
||||
@@ -241,7 +192,6 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
});
|
||||
}
|
||||
|
||||
// get image if it doesn't already exist
|
||||
const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
|
||||
if (!await helper.FileExists(imagePath)) {
|
||||
const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
|
||||
@@ -249,37 +199,88 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
|
||||
const buffer = await imageResponse.arrayBuffer();
|
||||
await fs.writeFile(imagePath, Buffer.from(buffer));
|
||||
} else {
|
||||
console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
|
||||
log(`Error fetching ${item.productId}: ${item.productName} image: ${imageResponse.statusText}`);
|
||||
await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
|
||||
}
|
||||
}
|
||||
|
||||
// be nice to the API and not send too many requests in a short time
|
||||
await helper.Sleep(300);
|
||||
|
||||
}
|
||||
|
||||
start += size;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
const syncTcgplayer = async (database: DBInstance, cardSets: string[], allProductIds: Set<number>, log: Logger) => {
|
||||
const productLines = ["pokemon", "pokemon-japan"];
|
||||
|
||||
for (const productLine of productLines) {
|
||||
const d = {
|
||||
"algorithm": "sales_dismax", "from": 0, "size": 1,
|
||||
"filters": { "term": { "productLineName": [productLine] } },
|
||||
"settings": { "useFuzzySearch": false },
|
||||
};
|
||||
|
||||
const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(d),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`TCGPlayer setName aggregation failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const data = await response.json();
|
||||
|
||||
const setNames = data.results[0].aggregations.setName;
|
||||
for (const setName of setNames) {
|
||||
let processSet = true;
|
||||
if (cardSets.length > 0) {
|
||||
processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
|
||||
}
|
||||
if (processSet) {
|
||||
log(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`);
|
||||
await syncProductLine(database, productLine, "setName", setName.urlValue, allProductIds, log);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log('All TCGPlayer data synchronized successfully!');
|
||||
};
|
||||
|
||||
|
||||
export const runImport = async (opts: RunImportOptions = {}) => {
|
||||
const { sets = [], log = consoleLogger, runUpdateVariants = true, runCardUpsert = true } = opts;
|
||||
|
||||
await fs.rm('missing_images.log', { force: true });
|
||||
|
||||
// When no set filter is provided, skip productIds already in the cards table
|
||||
// (matches the CLI script's "no args" behavior).
|
||||
const allProductIds = sets.length === 0
|
||||
? new Set<number>(
|
||||
await db.select({ productId: schema.cards.productId }).from(schema.cards)
|
||||
.then(rows => rows.map(row => row.productId))
|
||||
)
|
||||
: new Set<number>();
|
||||
|
||||
await syncTcgplayer(db, sets, allProductIds, log);
|
||||
|
||||
if (runUpdateVariants) {
|
||||
log('Updating card variants...');
|
||||
await helper.UpdateVariants(db, log);
|
||||
}
|
||||
|
||||
if (runCardUpsert) {
|
||||
log('Reindexing "cards" collection...');
|
||||
await helper.upsertCardCollection(db, log);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// CLI entry point — preserves the original `tsx scripts/preload-tcgplayer.ts [set...]` usage.
|
||||
const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isCli) {
|
||||
const args = process.argv.slice(2);
|
||||
await runImport({ sets: args });
|
||||
await ClosePool();
|
||||
}
|
||||
|
||||
// clear the log file
|
||||
await fs.rm('missing_images.log', { force: true });
|
||||
let allProductIds = new Set();
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0) {
|
||||
allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
|
||||
await syncTcgplayer();
|
||||
}
|
||||
else {
|
||||
await syncTcgplayer(args);
|
||||
}
|
||||
|
||||
// update the card table with new/updated variants
|
||||
await helper.UpdateVariants(db);
|
||||
|
||||
// index the card updates
|
||||
await helper.upsertCardCollection(db);
|
||||
|
||||
await ClosePool();
|
||||
|
||||
Reference in New Issue
Block a user