From d15585a6683e37c9d74a6ee57606ab1bccd01a6d Mon Sep 17 00:00:00 2001 From: Pyronewbic Date: Mon, 25 May 2026 19:09:20 +0530 Subject: [PATCH 1/3] fix: handle stale cache missing lang field, await set metadata before ready Cached cards without lang field defaulted all sets to JP. Default undefined lang to EN. Await set metadata fetch before marking card DB ready so logos are available on first request. --- lib/cards/card-database.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cards/card-database.js b/lib/cards/card-database.js index 9a7f73d..3fd899e 100644 --- a/lib/cards/card-database.js +++ b/lib/cards/card-database.js @@ -309,6 +309,7 @@ export async function initCardDatabase() { if (cached && cached.length > 0) { cardIndex = cached; deriveSetTotals(cardIndex); + await setMetaPromise; cardDbReady = true; console.log(`Card database loaded from cache: ${cardIndex.length} cards`); @@ -434,8 +435,9 @@ export function getAllSets() { } const entry = setMap.get(code); entry.count++; - if (card.lang === "en") entry.enCount++; - else if (card.lang === "jp") entry.jpCount++; + const lang = card.lang || "en"; + if (lang === "en") entry.enCount++; + else if (lang === "jp") entry.jpCount++; if (card.imageUrl) { const num = parseInt(card.localId, 10) || 0; if (num > entry.bestNum) { From 4fb9884a064f2e2ce23773f9b37fae54e2cc1e27 Mon Sep 17 00:00:00 2001 From: Pyronewbic Date: Mon, 25 May 2026 19:14:22 +0530 Subject: [PATCH 2/3] fix: deduplicate EN/JA cards with zero-padded IDs in older sets TCGdex JA API uses zero-padded localIds (neo1-001) while EN uses unpadded (neo1-1). Normalize card IDs before dedup so JA duplicates are merged with EN cards instead of appearing as separate JP-only entries with no images. --- lib/cards/card-database.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/cards/card-database.js b/lib/cards/card-database.js index 3fd899e..171487d 100644 --- a/lib/cards/card-database.js +++ b/lib/cards/card-database.js @@ -241,19 +241,28 @@ export async function loadCardDatabaseFromCache() { } } +function normalizeCardId(id) { + const dash = id.indexOf("-"); + if (dash === -1) return id; + const prefix = id.slice(0, dash); + const suffix = id.slice(dash + 1).replace(/^0+/, "") || "0"; + return `${prefix}-${suffix}`; +} + function buildIndexFromApi(enCards, jaCards, rarityMap = new Map()) { const jaMap = new Map(); for (const card of jaCards) { - jaMap.set(card.id, card); + jaMap.set(normalizeCardId(card.id), card); } const seen = new Set(); const index = []; for (const card of enCards) { - seen.add(card.id); + const normId = normalizeCardId(card.id); + seen.add(normId); const setCode = parseSetCode(card.id); - const jaCard = jaMap.get(card.id); + const jaCard = jaMap.get(normId); index.push({ id: card.id, name: card.name, @@ -267,7 +276,7 @@ function buildIndexFromApi(enCards, jaCards, rarityMap = new Map()) { } for (const card of jaCards) { - if (seen.has(card.id)) continue; + if (seen.has(normalizeCardId(card.id))) continue; const setCode = parseSetCode(card.id); const enName = translateJaName(card.name); index.push({ From 885c16afce7e4739ca945aa8868534bd4f614b20 Mon Sep 17 00:00:00 2001 From: Pyronewbic Date: Mon, 25 May 2026 19:25:17 +0530 Subject: [PATCH 3/3] feat: on-demand card DB sync, remove TTL-based refresh on startup Startup now always loads from Firestore cache without TTL expiry. No more background TCGdex re-fetch on every cold start. New POST /api/card-database/sync (owner-only) compares cached set codes against TCGdex and only rebuilds when new sets are detected. Supports ?force=true to bypass the check. --- CLAUDE.md | 3 +- api.js | 16 +++++++- lib/cards/card-database.js | 80 +++++++++++++++++++++++++++++--------- test/api-test.js | 5 +++ 4 files changed, 82 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c102ad0..4764a06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,7 @@ Strict palette — no deviations: ## Architecture decisions -- **Caching:** Firestore only, no Redis. Stale-while-revalidate on active listings. TCGdex card DB cached (24h TTL). PSA negative caching (7 days). +- **Caching:** Firestore only, no Redis. Stale-while-revalidate on active listings. TCGdex card DB cached (no TTL, synced on-demand via `POST /api/card-database/sync`). PSA negative caching (7 days). - **CORS:** Wildcard `*`. API key is the access control layer. - **Auth:** Owner `CC_LIVE_` (60/min), sandbox (5/min), developer keys (per-key rate limit enforced), demo `?demo=true` (360/min). Local: no auth. - **authMiddleware** checks owner → sandbox → JWT (Google OAuth) → Firestore developer keys (30s cache). `apiAuthMiddleware` adds demo bypass. @@ -100,6 +100,7 @@ Strict palette — no deviations: - `/api/analytics` — request analytics (owner-only) - `/api/grading-dataset/stats` — ML dataset collection stats (owner-only) - `/api/security/events` — RASP security event log (owner-only) +- `POST /api/card-database/sync` — sync card DB from TCGdex if new sets detected (owner-only, `?force=true` to force) - `/auth/google` — Google OAuth → JWT ## Free tier strategy diff --git a/api.js b/api.js index 3d3ac92..488d1ee 100644 --- a/api.js +++ b/api.js @@ -26,7 +26,7 @@ import { saveGradedImages } from "./lib/cards/grading-dataset.js"; import { verifyGoogleToken, generateJwt, verifyJwt } from "./lib/auth/auth.js"; import { seedFromTCGPlayer } from "./lib/sources/tcgplayer.js"; import { getOrCreateCard, findCardByQuery, parseCardIdentity, resolveCardIdToQuery, SET_NAME_MAP } from "./lib/cards/card-identity.js"; -import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js"; +import { initCardDatabase, searchCards, syncCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js"; import { raspMiddleware, getSecurityEvents } from "./lib/security/rasp.js"; import { fileURLToPath } from "url"; import path from "path"; @@ -905,6 +905,18 @@ app.get("/api/sets/:setCode", requireCardDb, (req, res) => { res.json(result); }); +// POST /api/card-database/sync +app.post("/api/card-database/sync", apiAuthMiddleware, async (req, res) => { + if (req.authTier !== "owner") return res.status(403).json({ error: "Owner only" }); + try { + const force = req.query.force === "true"; + const result = await syncCardDatabase({ force }); + res.json(result); + } catch (err) { + res.status(500).json({ error: "Sync failed" }); + } +}); + // ============ V1 API — Drop Intelligence ============ async function authMiddleware(req, res, next) { @@ -2227,7 +2239,7 @@ app.post("/api/track-prices", authMiddleware, async (req, res) => { } catch {} } - refreshCardDatabase().catch(() => {}); + syncCardDatabase().catch(() => {}); res.json({ tracked: results.length, results, portfolioSnapshots, portfolioWarmed, portfolioCardsTracked: portfolioQueries.length, frequencyWarmed }); }); diff --git a/lib/cards/card-database.js b/lib/cards/card-database.js index 171487d..390f9d1 100644 --- a/lib/cards/card-database.js +++ b/lib/cards/card-database.js @@ -168,7 +168,6 @@ function buildImageUrl(image) { } const CACHE_COLLECTION = "card-database-cache"; -const CACHE_TTL_MS = 24 * 60 * 60 * 1000; const CHUNK_SIZE = 5000; function getCacheDb() { @@ -198,9 +197,11 @@ export async function saveCardDatabaseToCache(index) { updatedAt: Date.now(), }); } + const setCodes = [...new Set(index.map(c => c.setCode).filter(Boolean))].sort(); batch.set(fs.collection(CACHE_COLLECTION).doc("meta"), { totalCards: index.length, totalChunks: chunks.length, + setCodes, updatedAt: Date.now(), }); await batch.commit(); @@ -214,7 +215,6 @@ export async function loadCardDatabaseFromCache() { const metaDoc = await fs.collection(CACHE_COLLECTION).doc("meta").get(); if (!metaDoc.exists) return null; const meta = metaDoc.data(); - if (Date.now() - meta.updatedAt > CACHE_TTL_MS) return null; const chunks = meta.totalChunks; const docs = []; @@ -321,16 +321,6 @@ export async function initCardDatabase() { await setMetaPromise; cardDbReady = true; console.log(`Card database loaded from cache: ${cardIndex.length} cards`); - - fetchAndBuildIndex().then(async (freshIndex) => { - if (freshIndex && freshIndex.length > 0) { - cardIndex = freshIndex; - deriveSetTotals(cardIndex); - console.log(`Card database refreshed from TCGdex: ${cardIndex.length} cards`); - await saveCardDatabaseToCache(freshIndex); - } - }).catch(() => {}); - return cardIndex.length; } @@ -338,6 +328,7 @@ export async function initCardDatabase() { if (freshIndex && freshIndex.length > 0) { cardIndex = freshIndex; deriveSetTotals(cardIndex); + await setMetaPromise; cardDbReady = true; console.log(`Card database loaded from TCGdex: ${cardIndex.length} cards`); saveCardDatabaseToCache(freshIndex).catch(() => {}); @@ -352,15 +343,66 @@ export function isCardDatabaseReady() { return cardDbReady; } -export async function refreshCardDatabase() { +async function getCachedSetCodes() { + const fs = getCacheDb(); + if (!fs) return []; + try { + const metaDoc = await fs.collection(CACHE_COLLECTION).doc("meta").get(); + if (!metaDoc.exists) return []; + return metaDoc.data().setCodes || []; + } catch { + return []; + } +} + +async function getTcgdexSetCodes() { + const sets = new Set(); + for (const lang of ["en", "ja"]) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + const res = await fetch(`https://api.tcgdex.net/v2/${lang}/sets`, { signal: controller.signal }); + clearTimeout(timer); + if (!res.ok) continue; + const data = await res.json(); + for (const s of data) { + if (s.id) sets.add(s.id.toLowerCase()); + } + } catch {} + } + return [...sets]; +} + +export async function syncCardDatabase({ force = false } = {}) { + const [cachedCodes, remoteCodes] = await Promise.all([ + getCachedSetCodes(), + getTcgdexSetCodes(), + ]); + + const cachedSet = new Set(cachedCodes.map(c => c.toLowerCase())); + const newSets = remoteCodes.filter(c => !cachedSet.has(c)); + + if (!force && newSets.length === 0) { + return { synced: false, reason: "no new sets", cachedSets: cachedCodes.length, remoteSets: remoteCodes.length }; + } + const freshIndex = await fetchAndBuildIndex(); - if (freshIndex && freshIndex.length > 0) { - cardIndex = freshIndex; - console.log(`Card database refreshed from TCGdex: ${cardIndex.length} cards`); - saveCardDatabaseToCache(freshIndex).catch(() => {}); - return cardIndex.length; + if (!freshIndex || freshIndex.length === 0) { + return { synced: false, reason: "TCGdex fetch failed" }; } - return cardIndex.length; + + cardIndex = freshIndex; + deriveSetTotals(cardIndex); + await saveCardDatabaseToCache(freshIndex); + console.log(`Card database synced: ${cardIndex.length} cards, ${newSets.length} new sets`); + + return { + synced: true, + totalCards: cardIndex.length, + newSets, + cachedSets: cachedCodes.length, + remoteSets: remoteCodes.length, + }; } export function getCardCount() { diff --git a/test/api-test.js b/test/api-test.js index e845b70..d0098e7 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -947,6 +947,11 @@ async function run() { } }); + await test("POST /api/card-database/sync requires owner key", async () => { + const { res } = await json("/api/card-database/sync", { method: "POST" }); + assert(res.status === 403, `expected 403, got ${res.status}`); + }); + // ── Price trend ── console.log("\n\x1b[1m=== price trend ===\x1b[0m");