fix: lang filter and missing set logos on cold start

CLAUDE.md

@@ -46,7 +46,7 @@ Strict palette — no deviations:
 
 ## Architecture decisions
 
-- **Caching:** Firestore only, no Redis. Stale-while-revalidate on active listings. TCGdex card DB cached (24h TTL). PSA negative caching (7 days).
+- **Caching:** Firestore only, no Redis. Stale-while-revalidate on active listings. TCGdex card DB cached (no TTL, synced on-demand via `POST /api/card-database/sync`). PSA negative caching (7 days).
 - **CORS:** Wildcard `*`. API key is the access control layer.
 - **Auth:** Owner `CC_LIVE_` (60/min), sandbox (5/min), developer keys (per-key rate limit enforced), demo `?demo=true` (360/min). Local: no auth.
 - **authMiddleware** checks owner → sandbox → JWT (Google OAuth) → Firestore developer keys (30s cache). `apiAuthMiddleware` adds demo bypass.
@@ -100,6 +100,7 @@ Strict palette — no deviations:
 - `/api/analytics` — request analytics (owner-only)
 - `/api/grading-dataset/stats` — ML dataset collection stats (owner-only)
 - `/api/security/events` — RASP security event log (owner-only)
+- `POST /api/card-database/sync` — sync card DB from TCGdex if new sets detected (owner-only, `?force=true` to force)
 - `/auth/google` — Google OAuth → JWT
 
 ## Free tier strategy

api.js

@@ -26,7 +26,7 @@ import { saveGradedImages } from "./lib/cards/grading-dataset.js";
 import { verifyGoogleToken, generateJwt, verifyJwt } from "./lib/auth/auth.js";
 import { seedFromTCGPlayer } from "./lib/sources/tcgplayer.js";
 import { getOrCreateCard, findCardByQuery, parseCardIdentity, resolveCardIdToQuery, SET_NAME_MAP } from "./lib/cards/card-identity.js";
-import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js";
+import { initCardDatabase, searchCards, syncCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js";
 import { raspMiddleware, getSecurityEvents } from "./lib/security/rasp.js";
 import { fileURLToPath } from "url";
 import path from "path";
@@ -905,6 +905,18 @@ app.get("/api/sets/:setCode", requireCardDb, (req, res) => {
   res.json(result);
 });
 
+// POST /api/card-database/sync
+app.post("/api/card-database/sync", apiAuthMiddleware, async (req, res) => {
+  if (req.authTier !== "owner") return res.status(403).json({ error: "Owner only" });
+  try {
+    const force = req.query.force === "true";
+    const result = await syncCardDatabase({ force });
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: "Sync failed" });
+  }
+});
+
 // ============ V1 API — Drop Intelligence ============
 
 async function authMiddleware(req, res, next) {
@@ -2227,7 +2239,7 @@ app.post("/api/track-prices", authMiddleware, async (req, res) => {
     } catch {}
   }
 
-  refreshCardDatabase().catch(() => {});
+  syncCardDatabase().catch(() => {});
   res.json({ tracked: results.length, results, portfolioSnapshots, portfolioWarmed, portfolioCardsTracked: portfolioQueries.length, frequencyWarmed });
 });
 

lib/cards/card-database.js

@@ -168,7 +168,6 @@ function buildImageUrl(image) {
 }
 
 const CACHE_COLLECTION = "card-database-cache";
-const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
 const CHUNK_SIZE = 5000;
 
 function getCacheDb() {
@@ -198,9 +197,11 @@ export async function saveCardDatabaseToCache(index) {
         updatedAt: Date.now(),
       });
     }
+    const setCodes = [...new Set(index.map(c => c.setCode).filter(Boolean))].sort();
     batch.set(fs.collection(CACHE_COLLECTION).doc("meta"), {
       totalCards: index.length,
       totalChunks: chunks.length,
+      setCodes,
       updatedAt: Date.now(),
     });
     await batch.commit();
@@ -214,7 +215,6 @@ export async function loadCardDatabaseFromCache() {
     const metaDoc = await fs.collection(CACHE_COLLECTION).doc("meta").get();
     if (!metaDoc.exists) return null;
     const meta = metaDoc.data();
-    if (Date.now() - meta.updatedAt > CACHE_TTL_MS) return null;
 
     const chunks = meta.totalChunks;
     const docs = [];
@@ -241,19 +241,28 @@ export async function loadCardDatabaseFromCache() {
   }
 }
 
+function normalizeCardId(id) {
+  const dash = id.indexOf("-");
+  if (dash === -1) return id;
+  const prefix = id.slice(0, dash);
+  const suffix = id.slice(dash + 1).replace(/^0+/, "") || "0";
+  return `${prefix}-${suffix}`;
+}
+
 function buildIndexFromApi(enCards, jaCards, rarityMap = new Map()) {
   const jaMap = new Map();
   for (const card of jaCards) {
-    jaMap.set(card.id, card);
+    jaMap.set(normalizeCardId(card.id), card);
   }
 
   const seen = new Set();
   const index = [];
 
   for (const card of enCards) {
-    seen.add(card.id);
+    const normId = normalizeCardId(card.id);
+    seen.add(normId);
     const setCode = parseSetCode(card.id);
-    const jaCard = jaMap.get(card.id);
+    const jaCard = jaMap.get(normId);
     index.push({
       id: card.id,
       name: card.name,
@@ -267,7 +276,7 @@ function buildIndexFromApi(enCards, jaCards, rarityMap = new Map()) {
   }
 
   for (const card of jaCards) {
-    if (seen.has(card.id)) continue;
+    if (seen.has(normalizeCardId(card.id))) continue;
     const setCode = parseSetCode(card.id);
     const enName = translateJaName(card.name);
     index.push({
@@ -309,25 +318,17 @@ export async function initCardDatabase() {
   if (cached && cached.length > 0) {
     cardIndex = cached;
     deriveSetTotals(cardIndex);
+    await setMetaPromise;
     cardDbReady = true;
     console.log(`Card database loaded from cache: ${cardIndex.length} cards`);
-
-    fetchAndBuildIndex().then(async (freshIndex) => {
-      if (freshIndex && freshIndex.length > 0) {
-        cardIndex = freshIndex;
-        deriveSetTotals(cardIndex);
-        console.log(`Card database refreshed from TCGdex: ${cardIndex.length} cards`);
-        await saveCardDatabaseToCache(freshIndex);
-      }
-    }).catch(() => {});
-
     return cardIndex.length;
   }
 
   const freshIndex = await fetchAndBuildIndex();
   if (freshIndex && freshIndex.length > 0) {
     cardIndex = freshIndex;
     deriveSetTotals(cardIndex);
+    await setMetaPromise;
     cardDbReady = true;
     console.log(`Card database loaded from TCGdex: ${cardIndex.length} cards`);
     saveCardDatabaseToCache(freshIndex).catch(() => {});
@@ -342,15 +343,66 @@ export function isCardDatabaseReady() {
   return cardDbReady;
 }
 
-export async function refreshCardDatabase() {
+async function getCachedSetCodes() {
+  const fs = getCacheDb();
+  if (!fs) return [];
+  try {
+    const metaDoc = await fs.collection(CACHE_COLLECTION).doc("meta").get();
+    if (!metaDoc.exists) return [];
+    return metaDoc.data().setCodes || [];
+  } catch {
+    return [];
+  }
+}
+
+async function getTcgdexSetCodes() {
+  const sets = new Set();
+  for (const lang of ["en", "ja"]) {
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), 15000);
+      const res = await fetch(`https://api.tcgdex.net/v2/${lang}/sets`, { signal: controller.signal });
+      clearTimeout(timer);
+      if (!res.ok) continue;
+      const data = await res.json();
+      for (const s of data) {
+        if (s.id) sets.add(s.id.toLowerCase());
+      }
+    } catch {}
+  }
+  return [...sets];
+}
+
+export async function syncCardDatabase({ force = false } = {}) {
+  const [cachedCodes, remoteCodes] = await Promise.all([
+    getCachedSetCodes(),
+    getTcgdexSetCodes(),
+  ]);
+
+  const cachedSet = new Set(cachedCodes.map(c => c.toLowerCase()));
+  const newSets = remoteCodes.filter(c => !cachedSet.has(c));
+
+  if (!force && newSets.length === 0) {
+    return { synced: false, reason: "no new sets", cachedSets: cachedCodes.length, remoteSets: remoteCodes.length };
+  }
+
   const freshIndex = await fetchAndBuildIndex();
-  if (freshIndex && freshIndex.length > 0) {
-    cardIndex = freshIndex;
-    console.log(`Card database refreshed from TCGdex: ${cardIndex.length} cards`);
-    saveCardDatabaseToCache(freshIndex).catch(() => {});
-    return cardIndex.length;
+  if (!freshIndex || freshIndex.length === 0) {
+    return { synced: false, reason: "TCGdex fetch failed" };
   }
-  return cardIndex.length;
+
+  cardIndex = freshIndex;
+  deriveSetTotals(cardIndex);
+  await saveCardDatabaseToCache(freshIndex);
+  console.log(`Card database synced: ${cardIndex.length} cards, ${newSets.length} new sets`);
+
+  return {
+    synced: true,
+    totalCards: cardIndex.length,
+    newSets,
+    cachedSets: cachedCodes.length,
+    remoteSets: remoteCodes.length,
+  };
 }
 
 export function getCardCount() {
@@ -434,8 +486,9 @@ export function getAllSets() {
     }
     const entry = setMap.get(code);
     entry.count++;
-    if (card.lang === "en") entry.enCount++;
-    else if (card.lang === "jp") entry.jpCount++;
+    const lang = card.lang || "en";
+    if (lang === "en") entry.enCount++;
+    else if (lang === "jp") entry.jpCount++;
     if (card.imageUrl) {
       const num = parseInt(card.localId, 10) || 0;
       if (num > entry.bestNum) {

test/api-test.js

@@ -947,6 +947,11 @@ async function run() {
     }
   });
 
+  await test("POST /api/card-database/sync requires owner key", async () => {
+    const { res } = await json("/api/card-database/sync", { method: "POST" });
+    assert(res.status === 403, `expected 403, got ${res.status}`);
+  });
+
   // ── Price trend ──
 
   console.log("\n\x1b[1m=== price trend ===\x1b[0m");