pokemon/scripts/preload-tcgplayer.ts

import 'dotenv/config';
import * as schema from '../src/db/schema.ts';
import { db, ClosePool } from '../src/db/index.ts';

import fs from "node:fs/promises";
import path from "node:path";
import chalk from 'chalk';
import * as helper from './pokemon-helper.ts';
//import util from 'util';


async function syncTcgplayer(cardSets:string[] = []) {

  const productLines = [ "pokemon", "pokemon-japan" ];

  // work from the available sets within the product line
  for (const productLine of productLines) {
    const d = {"algorithm":"sales_dismax","from":0,"size":1,"filters":{"term":{"productLineName":[productLine]}},"settings":{"useFuzzySearch":false}};

    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
      method: 'POST',
      headers: {'Content-Type': 'application/json',},
      body: JSON.stringify(d),
    });
    if (!response.ok) {
      console.error('Error notifying sync completion:', response.statusText);
      process.exit(1);
    }
    const data = await response.json();

    const setNames = data.results[0].aggregations.setName;
    for (const setName of setNames) {
      let processSet = true;
      if (cardSets.length > 0) {
        processSet = cardSets.some(set => setName.value.toLowerCase().includes(set.toLowerCase()));
      }
      if (processSet) {
        console.log(chalk.blue(`Syncing product line "${productLine}" with setName "${setName.urlValue}"...`));
        await syncProductLine(productLine, "setName", setName.urlValue);
      }
    }

  }

  console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
}


async function syncProductLine(productLine: string, field: string, fieldValue: string) {
  let start = 0;
  let size = 50;
  let total = 1000000;

  while (start < total) {
    console.log(` Fetching items ${start} to ${start + size} of ${total}...`);

    const d = {
          "algorithm":"sales_dismax",
          "from":start,
          "size":size,
          "filters":{
              "term":{"productLineName":[productLine], [field]:[fieldValue]} ,
              "range":{},
              "match":{}
          },
          "listingSearch":{
              "context":{"cart":{}},
              "filters":{"term":{
                  "sellerStatus":"Live",
                  "channelId":0
                  },
                  "range":{
                      "quantity":{"gte":1}
                  },
                  "exclude":{"channelExclusion":0}
              }
          },
          "context":{
              "cart":{},
              "shippingCountry":"US",
              "userProfile":{}
          },
          "settings":{
              "useFuzzySearch":false,
              "didYouMean":{}
          },
          "sort":{}
      };

    //console.log(util.inspect(d, { depth: null }));
    //process.exit(1);

    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(d),
    });

    if (!response.ok) {
      console.error('Error notifying sync completion:', response.statusText);
      process.exit(1);
    }

    const data = await response.json();
    total = data.results[0].totalResults;

    for (const item of data.results[0].results) {

      // Check if productId already exists and skip if it does (to avoid hitting the API too much)
      if (allProductIds.size > 0 && allProductIds.has(item.productId)) {
        continue;
      }

      console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));

      // Get product detail
      const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
        method: 'GET',
      });
      if (!detailResponse.ok) {
        console.error('Error fetching product details:', detailResponse.statusText);
        process.exit(1);
      }
      const detailData = await detailResponse.json();


      await db.insert(schema.tcgcards).values({
        productId: item.productId,
        productName: detailData.productName,
        //productName: cleanProductName(item.productName),
        rarityName: item.rarityName,
        productLineName: detailData.productLineName,
        productLineUrlName: detailData.productLineUrlName,
        productStatusId: detailData.productStatusId,
        productTypeId: detailData.productTypeId,
        productUrlName: detailData.productUrlName,
        setId: detailData.setId,
        shippingCategoryId: detailData.shippingCategoryId,
        sealed: detailData.sealed,
        sellerListable: detailData.sellerListable,
        foilOnly: detailData.foilOnly,
        attack1: item.customAttributes.attack1 || null,
        attack2: item.customAttributes.attack2 || null,
        attack3: item.customAttributes.attack3 || null,
        attack4: item.customAttributes.attack4 || null,
        cardType: item.customAttributes.cardType?.[0] || null,
        cardTypeB: item.customAttributes.cardTypeB || null,
        energyType: detailData.customAttributes.energyType?.[0] || null,
        flavorText: detailData.customAttributes.flavorText || null,
        hp: helper.GetNumberOrNull(item.customAttributes.hp),
        number: detailData.customAttributes.number || '',
        releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
        resistance: item.customAttributes.resistance || null,
        retreatCost: item.customAttributes.retreatCost || null,
        stage: item.customAttributes.stage || null,
        weakness: item.customAttributes.weakness || null,
        lowestPrice: detailData.lowestPrice,
        lowestPriceWithShipping: detailData.lowestPriceWithShipping,
        marketPrice: detailData.marketPrice,
        maxFulfillableQuantity: detailData.maxFulfillableQuantity,
        medianPrice: detailData.medianPrice,
        totalListings: item.totalListings,
        artist: detailData.formattedAttributes.Artist || null,
      }).onConflictDoUpdate({
        target: schema.tcgcards.productId,
        set: {
          productName: detailData.productName,
          //productName: cleanProductName(item.productName),
          rarityName: item.rarityName,
          productLineName: detailData.productLineName,
          productLineUrlName: detailData.productLineUrlName,
          productStatusId: detailData.productStatusId,
          productTypeId: detailData.productTypeId,
          productUrlName: detailData.productUrlName,
          setId: detailData.setId,
          shippingCategoryId: detailData.shippingCategoryId,
          sealed: detailData.sealed,
          sellerListable: detailData.sellerListable,
          foilOnly: detailData.foilOnly,
          attack1: item.customAttributes.attack1 || null,
          attack2: item.customAttributes.attack2 || null,
          attack3: item.customAttributes.attack3 || null,
          attack4: item.customAttributes.attack4 || null,
          cardType: item.customAttributes.cardType?.[0] || null,
          cardTypeB: item.customAttributes.cardTypeB || null,
          energyType: detailData.customAttributes.energyType?.[0] || null,
          flavorText: detailData.customAttributes.flavorText || null,
          hp: helper.GetNumberOrNull(item.customAttributes.hp),
          number: detailData.customAttributes.number || '',
          releaseDate: detailData.customAttributes.releaseDate ? new Date(detailData.customAttributes.releaseDate) : null,
          resistance: item.customAttributes.resistance || null,
          retreatCost: item.customAttributes.retreatCost || null,
          stage: item.customAttributes.stage || null,
          weakness: item.customAttributes.weakness || null,
          lowestPrice: detailData.lowestPrice,
          lowestPriceWithShipping: detailData.lowestPriceWithShipping,
          marketPrice: detailData.marketPrice,
          maxFulfillableQuantity: detailData.maxFulfillableQuantity,
          medianPrice: detailData.medianPrice,
          totalListings: item.totalListings,
          artist: detailData.formattedAttributes.Artist || null,
        },
      });

      console.log(`item: ${item.setId}\tdetail: ${detailData.setId}`);
      console.log(`item: ${item.setCode}\tdetail: ${detailData.setCode}`);
      console.log(`item: ${item.setName}\tdetail: ${detailData.setName}`);
      // set is...
      await db.insert(schema.sets).values({
        setId: detailData.setId,
        setCode: detailData.setCode,
        setName: detailData.setName,
        setUrlName: detailData.setUrlName,
      }).onConflictDoUpdate({
        target: schema.sets.setId,
        set: {
          setCode: detailData.setCode,
          setName: detailData.setName,
          setUrlName: detailData.setUrlName,
        },
      });

      // skus are...
      for (const skuItem of detailData.skus) {

        await db.insert(schema.skus).values({
          skuId: skuItem.sku,
          productId: detailData.productId,
          condition: skuItem.condition,
          language: skuItem.language,
          variant: skuItem.variant,
        }).onConflictDoUpdate({
          target: schema.skus.skuId,
          set: {
            condition: skuItem.condition,
            language: skuItem.language,
            variant: skuItem.variant,
          },
        });
      }

      // get image if it doesn't already exist
      const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
      if (!await helper.FileExists(imagePath)) {
        const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
        if (imageResponse.ok) {
          const buffer = await imageResponse.arrayBuffer();
          await fs.writeFile(imagePath, Buffer.from(buffer));
        } else {
          console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
          await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
        }
      }

      // be nice to the API and not send too many requests in a short time
      await helper.Sleep(300);

    }

    start += size;
  }
}

// clear the log file
await fs.rm('missing_images.log', { force: true });
let allProductIds = new Set();

const args = process.argv.slice(2);
if (args.length === 0) {
  allProductIds = new Set(await db.select({ productId: schema.cards.productId }).from(schema.cards).then(rows => rows.map(row => row.productId)));
  await syncTcgplayer();
}
else {
  await syncTcgplayer(args);
}

// update the card table with new/updated variants
await helper.UpdateVariants(db);

// index the card updates
helper.upsertCardCollection(db);

await ClosePool();