pokemon/scripts/preload-tcgplayer.ts

import 'dotenv/config';
import * as schema from '../src/db/schema.ts';
import { db, poolConnection } from '../src/db/index.ts';

import fs from "node:fs/promises";
import path from "node:path";
import chalk from 'chalk';
//import util from 'util';


async function syncTcgplayer() {

  const productLines = [
    { name: "pokemon", rarityName: ["Common", "Uncommon", "Promo", "Rare", "Ultra Rare", "Holo Rare", "Code Card", "Secret Rare", 
      "Illustration Rare", "Double Rare", "Shiny Holo Rare", "Special Illustration Rare", "Classic Collection", "Shiny Rare", 
      "Hyper Rare", "Unconfirmed", "ACE SPEC Rare", "Prism Rare", "Radiant Rare", "Rare BREAK", "Rare Ace", "Shiny Ultra Rare", 
      "Amazing Rare", "Mega Attack Rare", "Mega Hyper Rare", "Black White Rare"
    ] },
    { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
  ];

  for (const productLine of productLines) {
    for (const [key, values] of Object.entries(productLine)) {
      if (key === "name") continue;
      for (const value of values) {
        console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
        await syncProductLine(productLine.name, key, value);
      }
    }
  }

  console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
}

function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

function cleanProductName(name: string): string {
  // remove TCGPlayer crap
  name = name.replace(/ - .*$/, '');
  name = name.replace(/ \[.*\]/, '');
  name = name.replace(/ \(.*\)/, '');
  return name.trim();
}

async function fileExists(path: string): Promise<boolean> {
  try {
    await fs.access(path);
    return true;
  } catch {
    return false;
  }
}

async function syncProductLine(productLine: string, field: string, fieldValue: string) {
  let start = 0;
  let size = 50;
  let total = 1000000;

  while (start < total) {
    console.log(` Fetching items ${start} to ${start + size} of ${total}...`);

    const d = {
          "algorithm":"sales_dismax",
          "from":start,
          "size":size,
          "filters":{
              "term":{"productLineName":[productLine], [field]:[fieldValue]} ,
              "range":{},
              "match":{}
          },
          "listingSearch":{
              "context":{"cart":{}},
              "filters":{"term":{
                  "sellerStatus":"Live",
                  "channelId":0
                  },
                  "range":{
                      "quantity":{"gte":1}
                  },
                  "exclude":{"channelExclusion":0}
              }
          },
          "context":{
              "cart":{},
              "shippingCountry":"US",
              "userProfile":{}
          },
          "settings":{
              "useFuzzySearch":false,
              "didYouMean":{}
          },
          "sort":{}
      };

    //console.log(util.inspect(d, { depth: null }));
    //process.exit(1);

    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(d),
    });

    if (!response.ok) {
      console.error('Error notifying sync completion:', response.statusText);
      process.exit(1);
    }

    const data = await response.json();
    total = data.results[0].totalResults;

    for (const item of data.results[0].results) {
      console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));

      // Get product detail
      const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
        method: 'GET',
      });
      if (!detailResponse.ok) {
        console.error('Error fetching product details:', detailResponse.statusText);
        process.exit(1);
      }
      const detailData = await detailResponse.json();


      await db.insert(schema.cards).values({
        productId: item.productId,
        originalProductName: item.productName,
        productName: cleanProductName(item.productName),
        rarityName: item.rarityName,
        productLineName: item.productLineName,
        productLineUrlName: item.productLineUrlName,
        productStatusId: item.productStatusId,
        productTypeId: item.productTypeId,
        productUrlName: item.productUrlName,
        setId: item.setId,
        shippingCategoryId: item.shippingCategoryId,
        sealed: item.sealed,
        sellerListable: item.sellerListable,
        foilOnly: item.foilOnly,
        attack1: item.customAttributes.attack1 || null,
        attack2: item.customAttributes.attack2 || null,
        attack3: item.customAttributes.attack3 || null,
        attack4: item.customAttributes.attack4 || null,
        cardType: item.customAttributes.cardType?.[0] || null,
        cardTypeB: item.customAttributes.cardTypeB || null,
        energyType: item.customAttributes.energyType?.[0] || null,
        flavorText: item.customAttributes.flavorText || null,
        hp: item.customAttributes.hp || 0,
        number: item.customAttributes.number || '',
        releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
        resistance: item.customAttributes.resistance || null,
        retreatCost: item.customAttributes.retreatCost || null,
        stage: item.customAttributes.stage || null,
        weakness: item.customAttributes.weakness || null,
        lowestPrice: item.lowestPrice,
        lowestPriceWithShipping: item.lowestPriceWithShipping,
        marketPrice: item.marketPrice,
        maxFulfillableQuantity: item.maxFulfillableQuantity,
        medianPrice: item.medianPrice,
        totalListings: item.totalListings,
        Artist: detailData.formattedAttributes.Artist || null,
      }).onDuplicateKeyUpdate({
        set: {
          originalProductName: item.productName,
          productName: cleanProductName(item.productName),
          rarityName: item.rarityName,
          productLineName: item.productLineName,
          productLineUrlName: item.productLineUrlName,
          productStatusId: item.productStatusId,
          productTypeId: item.productTypeId,
          productUrlName: item.productUrlName,
          setId: item.setId,
          shippingCategoryId: item.shippingCategoryId,
          sealed: item.sealed,
          sellerListable: item.sellerListable,
          foilOnly: item.foilOnly,
          attack1: item.customAttributes.attack1 || null,
          attack2: item.customAttributes.attack2 || null,
          attack3: item.customAttributes.attack3 || null,
          attack4: item.customAttributes.attack4 || null,
          cardType: item.customAttributes.cardType?.[0] || null,
          cardTypeB: item.customAttributes.cardTypeB || null,
          energyType: item.customAttributes.energyType?.[0] || null,
          flavorText: item.customAttributes.flavorText || null,
          hp: item.customAttributes.hp || 0,
          number: item.customAttributes.number || '',
          releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
          resistance: item.customAttributes.resistance || null,
          retreatCost: item.customAttributes.retreatCost || null,
          stage: item.customAttributes.stage || null,
          weakness: item.customAttributes.weakness || null,
          lowestPrice: item.lowestPrice,
          lowestPriceWithShipping: item.lowestPriceWithShipping,
          marketPrice: item.marketPrice,
          maxFulfillableQuantity: item.maxFulfillableQuantity,
          medianPrice: item.medianPrice,
          totalListings: item.totalListings,
          Artist: detailData.formattedAttributes.Artist || null,
        },
      });


      // set is...
      await db.insert(schema.sets).values({
        setId: detailData.setId,
        setCode: detailData.setCode,
        setName: detailData.setName,
        setUrlName: detailData.setUrlName,
      }).onDuplicateKeyUpdate({
        set: {
          setCode: detailData.setCode,
          setName: detailData.setName,
          setUrlName: detailData.setUrlName,
        },
      });

      // skus are...
      for (const skuItem of detailData.skus) {

        await db.insert(schema.skus).values({
          skuId: skuItem.sku,
          productId: detailData.productId,
          condition: skuItem.condition,
          language: skuItem.language,
          variant: skuItem.variant,
        }).onDuplicateKeyUpdate({
          set: {
            condition: skuItem.condition,
            language: skuItem.language,
            variant: skuItem.variant,
          },
        });
      }

      // get image if it doesn't already exist
      const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
      if (!await fileExists(imagePath)) {
        const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
        if (imageResponse.ok) {
          const buffer = await imageResponse.arrayBuffer();
          await fs.writeFile(imagePath, Buffer.from(buffer));
        } else {
          console.error(chalk.yellow(`Error fetching ${item.productId}: ${item.productName} image:`, imageResponse.statusText));
          await fs.appendFile('missing_images.log', `${item.productId}: ${item.productName}\n`, 'utf-8');
        }
      }

      // be nice to the API and not send too many requests in a short time
      await sleep(100);

    }

    start += size;
  }
}

// clear the log file
await fs.rm('missing_images.log', { force: true });

await syncTcgplayer();
await poolConnection.end();