pokemon/scripts/preload-tcgplayer.ts

import 'dotenv/config';
import { drizzle } from 'drizzle-orm/mysql2';
import mysql from 'mysql2/promise';
import * as schema from '../src/db/schema.ts';
import fs from "node:fs/promises";
import path from "node:path";
import { eq } from 'drizzle-orm';
import chalk from 'chalk';
//import util from 'util';


async function syncTcgplayer() {

  const productLines = [
    { name: "pokemon", energyType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] },
    { name: "pokemon-japan", cardType: ["Water", "Fire", "Grass", "Lightning", "Psychic", "Fighting", "Darkness", "Metal", "Fairy", "Dragon", "Colorless", "Energy"] }
  ];

  for (const productLine of productLines) {
    for (const [key, values] of Object.entries(productLine)) {
      if (key === "name") continue;
      for (const value of values) {
        console.log(`Syncing product line "${productLine.name}" with ${key} "${value}"...`);
        await syncProductLineEnergyType(productLine.name, key, value);
      }
    }
  }

  console.log(chalk.green('✓ All TCGPlayer data synchronized successfully!'));
}

function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function fileExists(path: string): Promise<boolean> {
  try {
    await fs.access(path);
    return true;
  } catch {
    return false;
  }
}

async function syncProductLineEnergyType(productLine: string, field: string, fieldValue: string) {
  let start = 0;
  let size = 50;
  let total = 1000000;

  while (start < total) {
    console.log(` Fetching items ${start} to ${start + size} of ${total}...`);


    let d = {
          "algorithm":"sales_dismax",
          "from":start,
          "size":size,
          "filters":{
              "term":{"productLineName":[productLine]},
              "range":{},
              "match":{}
          },
          "listingSearch":{
              "context":{"cart":{}},
              "filters":{"term":{
                  "sellerStatus":"Live",
                  "channelId":0
                  },
                  "range":{
                      "quantity":{"gte":1}
                  },
                  "exclude":{"channelExclusion":0}
              }
          },
          "context":{
              "cart":{},
              "shippingCountry":"US",
              "userProfile":{}
          },
          "settings":{
              "useFuzzySearch":false,
              "didYouMean":{}
          },
          "sort":{}
      };
    d.filters.term[field] = [fieldValue];

    //console.log(util.inspect(d, { depth: null }));
    //process.exit(1);

    const response = await fetch('https://mp-search-api.tcgplayer.com/v1/search/request?q=&isList=false', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(d),
    });

    if (!response.ok) {
      console.error('Error notifying sync completion:', response.statusText);
      process.exit(1);
    }

    const data = await response.json();
    total = data.results[0].totalResults;

    //console.log(data);

    const poolConnection = mysql.createPool({
      uri: process.env.DATABASE_URL,
    });

    const db = drizzle(poolConnection, { schema, mode: 'default' });

    for (const item of data.results[0].results) {
      console.log(chalk.blue(` - ${item.productName} (ID: ${item.productId})`));

      await db.insert(schema.cards).values({
        productId: item.productId,
        productName: item.productName,
        rarityName: item.rarityName,
        productLineName: item.productLineName,
        productLineUrlName: item.productLineUrlName,
        productStatusId: item.productStatusId,
        productTypeId: item.productTypeId,
        productUrlName: item.productUrlName,
        setId: item.setId,
        shippingCategoryId: item.shippingCategoryId,
        sealed: item.sealed,
        sellerListable: item.sellerListable,
        foilOnly: item.foilOnly,
        attack1: item.customAttributes.attack1 || null,
        attack2: item.customAttributes.attack2 || null,
        attack3: item.customAttributes.attack3 || null,
        attack4: item.customAttributes.attack4 || null,
        cardType: item.customAttributes.cardType?.[0] || null,
        cardTypeB: item.customAttributes.cardTypeB || null,
        energyType: item.customAttributes.energyType?.[0] || null,
        flavorText: item.customAttributes.flavorText || null,
        hp: item.customAttributes.hp || 0,
        number: item.customAttributes.number || '',
        releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
        resistance: item.customAttributes.resistance || null,
        retreatCost: item.customAttributes.retreatCost || null,
        stage: item.customAttributes.stage || null,
        weakness: item.customAttributes.weakness || null,
        lowestPrice: item.lowestPrice,
        lowestPriceWithShipping: item.lowestPriceWithShipping,
        marketPrice: item.marketPrice,
        maxFulfillableQuantity: item.maxFulfillableQuantity,
        medianPrice: item.medianPrice,
        totalListings: item.totalListings,
      }).onDuplicateKeyUpdate({
        set: {
          productName: item.productName,
          rarityName: item.rarityName,
          productLineName: item.productLineName,
          productLineUrlName: item.productLineUrlName,
          productStatusId: item.productStatusId,
          productTypeId: item.productTypeId,
          productUrlName: item.productUrlName,
          setId: item.setId,
          shippingCategoryId: item.shippingCategoryId,
          sealed: item.sealed,
          sellerListable: item.sellerListable,
          foilOnly: item.foilOnly,
          attack1: item.customAttributes.attack1 || null,
          attack2: item.customAttributes.attack2 || null,
          attack3: item.customAttributes.attack3 || null,
          attack4: item.customAttributes.attack4 || null,
          cardType: item.customAttributes.cardType?.[0] || null,
          cardTypeB: item.customAttributes.cardTypeB || null,
          energyType: item.customAttributes.energyType?.[0] || null,
          flavorText: item.customAttributes.flavorText || null,
          hp: item.customAttributes.hp || 0,
          number: item.customAttributes.number || '',
          releaseDate: item.customAttributes.releaseDate ? new Date(item.customAttributes.releaseDate) : null,
          resistance: item.customAttributes.resistance || null,
          retreatCost: item.customAttributes.retreatCost || null,
          stage: item.customAttributes.stage || null,
          weakness: item.customAttributes.weakness || null,
          lowestPrice: item.lowestPrice,
          lowestPriceWithShipping: item.lowestPriceWithShipping,
          marketPrice: item.marketPrice,
          maxFulfillableQuantity: item.maxFulfillableQuantity,
          medianPrice: item.medianPrice,
          totalListings: item.totalListings,
        },
      });

      // before we fetch details, check if the card already exists in the skus table with a recent calculatedAt date. If it does, we can skip fetching details and pricing for this card to reduce API calls.
      const existingSkus = await db.select().from(schema.skus).where(eq(schema.skus.productId, item.productId));
      const hasRecentSku = existingSkus.some(sku => sku.calculatedAt && (new Date().getTime() - new Date(sku.calculatedAt).getTime()) < 7 * 24 * 60 * 60 * 1000);
      if (hasRecentSku) {
        console.log(chalk.blue('  Skipping details and pricing fetch since we have recent SKU data'));
        await sleep(100);
        continue;
      }

      // Get product detail
      const detailResponse = await fetch(`https://mp-search-api.tcgplayer.com/v2/product/${item.productId}/details`, {
        method: 'GET',
      });

      if (!detailResponse.ok) {
        console.error('Error fetching product details:', detailResponse.statusText);
        process.exit(1);
      }

      const detailData = await detailResponse.json();

      await db.insert(schema.sets).values({
        setId: detailData.setId,
        setCode: detailData.setCode,
        setName: detailData.setName,
        setUrlName: detailData.setUrlName,
      }).onDuplicateKeyUpdate({
        set: {
          setCode: detailData.setCode,
          setName: detailData.setName,
          setUrlName: detailData.setUrlName,
        },
      });

      // skus are...
      const skuArray = detailData.skus.map((sku: any) => sku.sku);
      //console.log(detailData.skus);
      //console.log(skuArray);
      // get pricing for skus
      const skuResponse = await fetch('https://mpgateway.tcgplayer.com/v1/pricepoints/marketprice/skus/search', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({ skuIds: skuArray }),
      });
      
      if (!skuResponse.ok) {
        console.error('Error fetching SKU pricing:', skuResponse.statusText);
        process.exit(1);
      }

      const skuData = await skuResponse.json();

      let skuMap = new Map();
      for (const skuItem of skuData) {
        skuMap.set(skuItem.skuId, skuItem);
      }

      for (const skuItem of detailData.skus) {
        const pricing = skuMap.get(skuItem.sku);
        //console.log(pricing);

        await db.insert(schema.skus).values({
          skuId: skuItem.sku,
          productId: detailData.productId,
          condition: skuItem.condition,
          language: skuItem.language,
          variant: skuItem.variant,
          calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
          highestPrice: pricing?.highestPrice || null,
          lowestPrice: pricing?.lowestPrice || null,
          marketPrice: pricing?.marketPrice || null,
          priceCount: pricing?.priceCount || 0,
        }).onDuplicateKeyUpdate({
          set: {
            condition: skuItem.condition,
            language: skuItem.language,
            variant: skuItem.variant,
            calculatedAt: pricing?.calculatedAt ? new Date(pricing.calculatedAt) : null,
            highestPrice: pricing?.highestPrice || null,
            lowestPrice: pricing?.lowestPrice || null,
            marketPrice: pricing?.marketPrice || null,
            priceCount: pricing?.priceCount || 0,
          },
        });
      }

      // get image if it doesn't already exist
      const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
      if (!await fileExists(imagePath)) {
        const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
        if (imageResponse.ok) {
          const buffer = await imageResponse.arrayBuffer();
          await fs.writeFile(imagePath, Buffer.from(buffer));
        } else {
          console.error('Error fetching product image:', imageResponse.statusText);
        }
      }

      // be nice to the API and not send too many requests in a short time
      await sleep(100);

    }


    await poolConnection.end();

    start += size;
  }
}


syncTcgplayer();