adsense verification, remove latest sales table, add new search mechanic (weight, synonyms), fix low volatility (NaN%)

.gitignore

@@ -26,6 +26,9 @@ pnpm-debug.log*
 # imges from tcgplayer
 public/cards/*
 
+# static assets
+/static/
+
 # anything test
 test.*
 

scripts/list-missing-images.ts

@@ -12,7 +12,7 @@ async function findMissingImages() {
       .where(sql`${schema.tcgcards.sealed} = false`);
     const missingImages: string[] = [];
     for (const card of cards) {
-        const imagePath = path.join(process.cwd(), 'public', 'cards', `${card.productId}.jpg`);
+        const imagePath = path.join(process.cwd(), 'static', 'cards', `${card.productId}.jpg`);
         try {
             await fs.access(imagePath);
         } catch (err) {

scripts/pokemon-helper.ts

@@ -54,6 +54,7 @@ export const createCardCollection = async () => {
             { name: 'productLineName', type: 'string', facet: true },
             { name: 'rarityName', type: 'string', facet: true },
             { name: 'setName', type: 'string', facet: true },
+            { name: 'setCode', type: 'string' },
             { name: 'cardType', type: 'string', facet: true },
             { name: 'energyType', type: 'string', facet: true },
             { name: 'number', type: 'string', sort: true },
@@ -94,7 +95,15 @@ export const upsertCardCollection = async (db:DBInstance) => {
         with: { set: true, tcgdata: true, prices: true },
     });
     await client.collections('cards').documents().import(pokemon.map(card => {
-        const marketPrice = card.tcgdata?.marketPrice ? DollarToInt(card.tcgdata.marketPrice) : null;
+        // Use the NM SKU price matching the card's variant (kept fresh by syncPrices)
+        // Fall back to any NM sku, then to tcgdata price
+        const nmSku = card.prices.find(p => p.condition === 'Near Mint' && p.variant === card.variant)
+                   ?? card.prices.find(p => p.condition === 'Near Mint');
+        const marketPrice = nmSku?.marketPrice
+            ? DollarToInt(nmSku.marketPrice)
+            : card.tcgdata?.marketPrice
+                ? DollarToInt(card.tcgdata.marketPrice)
+                : null;
 
         return {
             id: card.cardId.toString(),
@@ -105,12 +114,13 @@ export const upsertCardCollection = async (db:DBInstance) => {
             productLineName: card.productLineName,
             rarityName: card.rarityName,
             setName: card.set?.setName || "",
+            setCode: card.set?.setCode || "",
             cardType: card.cardType || "",
             energyType: card.energyType || "",
             number: card.number,
             Artist: card.artist || "",
             sealed: card.sealed,
-            content: [card.productName, card.productLineName, card.set?.setName || "", card.number, card.rarityName, card.artist || ""].join(' '),
+            content: [card.productName, card.productLineName, card.set?.setName || "", card.set?.setCode || "", card.number, card.rarityName, card.artist || ""].join(' '),
             releaseDate: card.tcgdata?.releaseDate ? Math.floor(new Date(card.tcgdata.releaseDate).getTime() / 1000) : 0,
             ...(marketPrice !== null && { marketPrice }),
             sku_id: card.prices.map(price => price.skuId.toString())

scripts/preload-tcgplayer.ts

@@ -242,7 +242,7 @@ async function syncProductLine(productLine: string, field: string, fieldValue: s
       }
 
       // get image if it doesn't already exist
-      const imagePath = path.join(process.cwd(), 'public', 'cards', `${item.productId}.jpg`);
+      const imagePath = path.join(process.cwd(), 'static', 'cards', `${item.productId}.jpg`);
       if (!await helper.FileExists(imagePath)) {
         const imageResponse = await fetch(`https://tcgplayer-cdn.tcgplayer.com/product/${item.productId}_in_1000x1000.jpg`);
         if (imageResponse.ok) {
@@ -280,6 +280,6 @@ else {
 await helper.UpdateVariants(db);
 
 // index the card updates
-helper.upsertCardCollection(db);
+await helper.upsertCardCollection(db);
 
 await ClosePool();

scripts/sync-prices.ts

@@ -154,6 +154,7 @@ const updateLatestSales = async (updatedCards: Set<number>) => {
 const start = Date.now();
 const updatedCards = await syncPrices();
 await helper.upsertSkuCollection(db);
+await helper.upsertCardCollection(db);  
 //console.log(updatedCards);
 //console.log(updatedCards.size);
 //await updateLatestSales(updatedCards);

src/layouts/Main.astro

@@ -16,6 +16,7 @@ const { title } = Astro.props;
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="htmx-config" content='{"historyCacheSize": 50}'/>
+    <meta name="google-adsense-account" content="ca-pub-1140571217687341">
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <link rel="icon" href="/favicon.ico" />
     <title>{title}</title>

src/middleware.ts

@@ -1,17 +1,45 @@
-// src/middleware.ts
+import { clerkMiddleware, createRouteMatcher, clerkClient } from '@clerk/astro/server';
-import { clerkMiddleware, createRouteMatcher } from '@clerk/astro/server';
-import type { AstroMiddlewareRequest, AstroMiddlewareResponse } from 'astro';
 
-const isProtectedRoute = createRouteMatcher([
+const isProtectedRoute = createRouteMatcher(['/pokemon']);
-  '/pokemon',
+const isAdminRoute = createRouteMatcher(['/admin']);
-]);
 
-export const onRequest = clerkMiddleware((auth, context) => {
+const TARGET_ORG_ID = "org_3Baav9czkRLLlC7g89oJWqRRulK";
-  const { isAuthenticated, redirectToSignIn } = auth()
+
+export const onRequest = clerkMiddleware(async (auth, context) => {
+  const { isAuthenticated, userId, redirectToSignIn } = auth();
 
   if (!isAuthenticated && isProtectedRoute(context.request)) {
-    // Add custom logic to run before redirecting
+    return redirectToSignIn();
+  }
 
-    return redirectToSignIn()
+  if (isAdminRoute(context.request)) {
+    if (!isAuthenticated || !userId) {
+      return redirectToSignIn();
+    }
+
+    try {
+      const client = await clerkClient(context); // pass context here
+      const memberships = await client.organizations.getOrganizationMembershipList({
+        organizationId: TARGET_ORG_ID,
+      });
+
+      console.log("Total memberships found:", memberships.data.length);
+      console.log("Current userId:", userId);
+      console.log("Memberships:", JSON.stringify(memberships.data.map(m => ({
+        userId: m.publicUserData?.userId,
+        role: m.role,
+      })), null, 2));
+
+      const userMembership = memberships.data.find(
+        (m) => m.publicUserData?.userId === userId
+      );
+
+      if (!userMembership || userMembership.role !== "org:admin") {
+        return context.redirect("/");
+      }
+    } catch (e) {
+      console.error("Clerk membership check failed:", e);
+      return context.redirect("/");
+    }
   }
 });

src/pages/admin.astro

@@ -0,0 +1,18 @@
+---
+export const prerender = false;
+import Layout from '../layouts/Main.astro';
+import NavItems from '../components/NavItems.astro';
+import NavBar from '../components/NavBar.astro';
+import Footer from '../components/Footer.astro';
+---
+<Layout title="Admin Panel">
+  <NavBar slot="navbar">
+    <NavItems slot="navItems" />
+  </NavBar>
+  <div class="row mb-4" slot="page">
+    <div class="col-12">
+      <h1>Admin Panel</h1>
+    </div>
+  </div>
+  <Footer slot="footer" />
+</Layout>

src/pages/api/upload.ts

@@ -0,0 +1,95 @@
+// src/pages/api/upload.ts
+import type { APIRoute } from 'astro';
+import { parse, stringify, transform } from 'csv';
+import { Readable } from 'stream';
+import { client } from '../../db/typesense';
+import chalk from 'chalk';
+import { db, ClosePool } from '../../db/index';
+
+// Define the transformation logic
+const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
+try {
+    // Specific query bsaed on tcgcollector CSV
+    const query = String(Object.values(row)[1]);
+    const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
+    const cardNumber = String(Object.values(row)[7]);
+    console.log(`${query} ${cardNumber} : ${setname}`);
+
+    // Use Typesense to find the card because we can easily use the combined fields
+    let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
+    if (cards.hits?.length === 0) {
+        // Try without card number
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
+    }
+    if (cards.hits?.length === 0) {
+        // Try without set name
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
+    }
+    if (cards.hits?.length === 0) {
+        // I give up, just output the values from the csv
+        console.log(chalk.red('  - not found'));
+        const newRow = { ...row };
+        newRow.Variant = '';
+        newRow.marketPrice = '';
+        this.push(newRow);
+    }
+    else {
+        for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
+            console.log(chalk.blue(`  - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
+            const variant = await db.query.cards.findFirst({
+                with: { prices: true, tcgdata: true },
+                where: { cardId: card.cardId }
+            });
+            const newRow = { ...row };
+            newRow.Variant = variant?.variant;
+            newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
+            this.push(newRow);
+        }
+    }
+
+    callback();
+} catch (error) {
+    callback(error);
+}
+});
+
+
+export const POST: APIRoute = async ({ request }) => {
+  try {
+    const formData = await request.formData();
+    const file = formData.get('file') as File;
+    const inputStream = Readable.from(file.stream());
+
+    if (!file) {
+      return new Response('No file uploaded', { status: 400 });
+    }
+
+    // Pipe the streams: Read -> Parse -> Transform -> Stringify -> Write
+    const outputStream = inputStream
+        .on('error', (error) => console.error('Input stream error:', error))
+        .pipe(parse({ columns: true, trim: true }))
+        .on('error', (error) => console.error('Parse error:', error))
+        .pipe(transformer)
+        .on('error', (error) => console.error('Transform error:', error))
+        .pipe(stringify({ header: true }))
+        .on('error', (error) => console.error('Stringify error:', error));
+
+    // outputStream.on('finish', () => {
+    //     ClosePool();
+    // }).on('error', (error) => {
+    //     ClosePool();
+    // });
+
+    
+    return new Response(outputStream as any, {
+        headers: {
+            'Content-Type': 'text/csv',
+            'Content-Disposition': 'attachment; filename=transformed.csv',
+        },
+    });
+
+  } catch (error) {
+    console.error('Error processing CSV stream:', error);
+    return new Response('Internal Server Error', { status: 500 });
+  }
+};

src/pages/myprices.astro

@@ -0,0 +1,26 @@
+---
+import Layout from '../layouts/Main.astro';
+import NavItems from '../components/NavItems.astro';
+import NavBar from '../components/NavBar.astro';
+import Footer from '../components/Footer.astro';
+
+---
+<Layout title="Rigid's App Thing">
+    <NavBar slot="navbar">
+        <NavItems slot="navItems" />
+    </NavBar>
+    <div class="row mb-4" slot="page">
+        <div class="col-12">
+            <h1>Rigid's App Thing</h1>
+            <p class="text-secondary">(working title)</p>
+        </div>
+        <div class="col-12">
+            <!-- src/components/FileUploader.astro -->
+            <form action="/api/upload" method="post" enctype="multipart/form-data">
+                <input type="file" name="file" accept=".csv" required />
+                <button type="submit">Upload CSV</button>
+            </form>
+        </div>
+    </div>
+    <Footer slot="footer" />
+</Layout>

src/pages/partials/card-modal.astro

@@ -46,7 +46,7 @@ const calculatedAt = (() => {
   const dates = card.prices
     .map(p => p.calculatedAt)
     .filter(d => d)
-    .map(d => new Date(d));
+    .map(d => new Date(d!));
   if (!dates.length) return null;
   return new Date(Math.max(...dates.map(d => d.getTime())));
 })();
@@ -88,13 +88,22 @@ for (const row of historyRows) {
 
 function computeVolatility(prices: number[]): { label: string; monthlyVol: number } {
   if (prices.length < 2) return { label: '—', monthlyVol: 0 };
+
   const returns: number[] = [];
   for (let i = 1; i < prices.length; i++) {
-    returns.push(Math.log(prices[i] / prices[i - 1]));
+    const ratio = prices[i] / prices[i - 1];
+    if (!isFinite(ratio) || ratio <= 0) continue; // skip bad ratios
+    returns.push(Math.log(ratio));
   }
+
+  if (returns.length < 2) return { label: '—', monthlyVol: 0 }; // ← key fix
+
   const mean = returns.reduce((a, b) => a + b, 0) / returns.length;
   const variance = returns.reduce((sum, r) => sum + (r - mean) ** 2, 0) / (returns.length - 1);
   const monthlyVol = Math.sqrt(variance) * Math.sqrt(30);
+
+  if (!isFinite(monthlyVol)) return { label: '—', monthlyVol: 0 }; // safety net
+
   const label = monthlyVol >= 0.30 ? 'High'
               : monthlyVol >= 0.15 ? 'Medium'
               : 'Low';
@@ -201,11 +210,11 @@ const altSearchUrl = (card: any) => {
                     data-name={card?.productName}
                   >
                     <img
-                      src={`/cards/${card?.productId}.jpg`}
+                      src={`/static/cards/${card?.productId}.jpg`}
                       class="card-image w-100 img-fluid rounded-4"
                       alt={card?.productName}
                       crossorigin="anonymous"
-                      onerror="this.onerror=null; this.src='/cards/default.jpg'; this.closest('.image-grow, .card-image-wrap')?.setAttribute('data-default','true')"
+                      onerror="this.onerror=null; this.src='/static/cards/default.jpg'; this.closest('.image-grow, .card-image-wrap')?.setAttribute('data-default','true')"
                       onclick="copyImage(this); dataLayer.push({'event': 'copiedImage'});"
                     />
                   </div>
@@ -310,7 +319,7 @@ const altSearchUrl = (card: any) => {
 
                           <!-- Table only — chart is outside the tab panes -->
                           <div class="w-100">
-                            <div class="alert alert-dark rounded p-2 mb-0 table-responsive">
+                            <div class="alert alert-dark rounded p-2 mb-0 table-responsive d-none">
                               <h6>Latest Verified Sales</h6>
                               <table class="table table-sm mb-0">
                                 <caption class="small">Filtered to remove mismatched language variants</caption>

src/pages/partials/cards.astro

@@ -48,15 +48,53 @@ const languageFilter = language === 'en' ? " && productLineName:=`Pokemon`"
 // synonyms alone (e.g. terms that need to match across multiple set names)
 // and rewrites them into a direct filter, clearing the query so it doesn't
 // also try to text-match against card names.
-const EREADER_SETS = ['Expedition Base Set', 'Aquapolis', 'Skyridge', 'Battle-e'];
+
-const EREADER_RE = /^(e-?reader|e reader)$/i;
+const ALIAS_FILTERS = [
+    // ── Era / set groupings ───────────────────────────────────────────────
+    { re: /^(e-?reader|e reader)$/i, field: 'setName',
+      values: ['Expedition Base Set', 'Aquapolis', 'Skyridge', 'Battle-e'] },
+
+    { re: /^neo$/i, field: 'setName',
+      values: ['Neo Genesis', 'Neo Discovery', 'Neo Revelation', 'Neo Destiny'] },
+
+    { re: /^(wotc|wizards)$/i, field: 'setName',
+      values: ['Base Set', 'Jungle', 'Fossil', 'Base Set 2', 'Team Rocket',
+               'Gym Heroes', 'Gym Challenge', 'Neo Genesis', 'Neo Discovery',
+               'Neo Revelation', 'Neo Destiny', 'Expedition Base Set',
+               'Aquapolis', 'Skyridge', 'Battle-e'] },
+
+    { re: /^(sun\s*(&|and)\s*moon|s(&|and)m|sm)$/i, field: 'setName',
+      values: ['Sun & Moon', 'Guardians Rising', 'Burning Shadows', 'Crimson Invasion',
+               'Ultra Prism', 'Forbidden Light', 'Celestial Storm', 'Dragon Majesty',
+               'Lost Thunder', 'Team Up', 'Unbroken Bonds', 'Unified Minds',
+               'Hidden Fates', 'Cosmic Eclipse', 'Detective Pikachu'] },
+
+    { re: /^(sword\s*(&|and)\s*shield|s(&|and)s|swsh)$/i, field: 'setName',
+      values: ['Sword & Shield', 'Rebel Clash', 'Darkness Ablaze', 'Vivid Voltage',
+               'Battle Styles', 'Chilling Reign', 'Evolving Skies', 'Fusion Strike',
+               'Brilliant Stars', 'Astral Radiance', 'Pokemon GO', 'Lost Origin',
+               'Silver Tempest', 'Crown Zenith'] },
+
+    // ── Card type shorthands ──────────────────────────────────────────────
+    { re: /^trainers?$/i,         field: 'cardType', values: ['Trainer'] },
+    { re: /^supporters?$/i,       field: 'cardType', values: ['Supporter'] },
+    { re: /^stadiums?$/i,         field: 'cardType', values: ['Stadium'] },
+    { re: /^items?$/i,            field: 'cardType', values: ['Item'] },
+    { re: /^(energys?|energies)$/i, field: 'cardType', values: ['Energy'] },
+
+    // ── Rarity shorthands ─────────────────────────────────────────────────
+    { re: /^promos?$/i, field: 'rarityName', values: ['Promo'] },
+];
 
 let resolvedQuery = query;
 let queryFilter = '';
 
-if (EREADER_RE.test(query.trim())) {
+for (const alias of ALIAS_FILTERS) {
+    if (alias.re.test(query.trim())) {
         resolvedQuery = '';
-    queryFilter = `setName:=[${EREADER_SETS.map(s => '`' + s + '`').join(',')}]`;
+        queryFilter = `${alias.field}:=[${alias.values.map(s => '`' + s + '`').join(',')}]`;
+        break;
+    }
 }
 
 const filters = Array.from(formData.entries())
@@ -118,7 +156,10 @@ if (start === 0) {
 const searchRequests = { searches: searchArray };
 const commonSearchParams = {
     q: resolvedQuery,
-    query_by: 'content,setName,productLineName,rarityName,energyType,cardType'
+    query_by: 'content,setName,setCode,productName',
+    query_by_weights: '10,6,8,9',
+    num_typos: '2,1,0,1',
+    prefix: 'true,true,false,false',
 };
 
 // use typesense to search for cards matching the query and return the productIds of the results
@@ -276,8 +317,8 @@ const facets = searchResults.results.slice(1).map((result: any) => {
 }
 
 {pokemon.length === 0 && (
-    <div id="notfound" hx-swap-oob="true">
+    <div id="notfound" class="mt-4 h6" hx-swap-oob="true">
-        Pokemon not found
+        No cards found! Please modify your search and try again.
     </div>
 )}
 
@@ -287,7 +328,7 @@ const facets = searchResults.results.slice(1).map((result: any) => {
             <div class="inventory-label pt-2">+/-</div>
         </div>
         <div class="card-trigger position-relative" data-card-id={card.cardId} hx-get={`/partials/card-modal?cardId=${card.cardId}`} hx-target="#cardModal" hx-trigger="click" data-bs-toggle="modal" data-bs-target="#cardModal" onclick="const cardTitle = this.querySelector('#cardImage').getAttribute('alt'); dataLayer.push({'event': 'virtualPageview', 'pageUrl': this.getAttribute('hx-get'), 'pageTitle': cardTitle, 'previousUrl': '/pokemon'});">
-            <div class="image-grow rounded-4 card-image" data-energy={card.energyType} data-rarity={card.rarityName} data-variant={card.variant} data-name={card.productName}><img src={`/cards/${card.productId}.jpg`} alt={card.productName} id="cardImage" loading="lazy" decoding="async" class="img-fluid rounded-4 mb-2 w-100" onerror="this.onerror=null; this.src='/cards/default.jpg'; this.closest('.image-grow')?.setAttribute('data-default','true')"/><span class="position-absolute top-50 start-0 d-inline medium-icon"><FirstEditionIcon edition={card?.variant} /></span>
+            <div class="image-grow rounded-4 card-image" data-energy={card.energyType} data-rarity={card.rarityName} data-variant={card.variant} data-name={card.productName}><img src={`/static/cards/${card.productId}.jpg`} alt={card.productName} id="cardImage" loading="lazy" decoding="async" class="img-fluid rounded-4 mb-2 w-100" onerror="this.onerror=null; this.src='/static/cards/default.jpg'; this.closest('.image-grow')?.setAttribute('data-default','true')"/><span class="position-absolute top-50 start-0 d-inline medium-icon"><FirstEditionIcon edition={card?.variant} /></span>
               <div class="holo-shine"></div>
               <div class="holo-glare"></div>
             </div>

tsconfig.json

@@ -1,5 +1,5 @@
 {
   "extends": "astro/tsconfigs/strict",
-  "include": [".astro/types.d.ts", "**/*"],
+  "include": [".astro/types.d.ts", "src/**/*"],
   "exclude": ["dist"]
 }