diff --git a/CLAUDE.md b/CLAUDE.md index c102ad0..4764a06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,7 @@ Strict palette — no deviations: ## Architecture decisions -- **Caching:** Firestore only, no Redis. Stale-while-revalidate on active listings. TCGdex card DB cached (24h TTL). PSA negative caching (7 days). +- **Caching:** Firestore only, no Redis. Stale-while-revalidate on active listings. TCGdex card DB cached (no TTL, synced on-demand via `POST /api/card-database/sync`). PSA negative caching (7 days). - **CORS:** Wildcard `*`. API key is the access control layer. - **Auth:** Owner `CC_LIVE_` (60/min), sandbox (5/min), developer keys (per-key rate limit enforced), demo `?demo=true` (360/min). Local: no auth. - **authMiddleware** checks owner → sandbox → JWT (Google OAuth) → Firestore developer keys (30s cache). `apiAuthMiddleware` adds demo bypass. @@ -100,6 +100,7 @@ Strict palette — no deviations: - `/api/analytics` — request analytics (owner-only) - `/api/grading-dataset/stats` — ML dataset collection stats (owner-only) - `/api/security/events` — RASP security event log (owner-only) +- `POST /api/card-database/sync` — sync card DB from TCGdex if new sets detected (owner-only, `?force=true` to force) - `/auth/google` — Google OAuth → JWT ## Free tier strategy diff --git a/api.js b/api.js index 3d3ac92..488d1ee 100644 --- a/api.js +++ b/api.js @@ -26,7 +26,7 @@ import { saveGradedImages } from "./lib/cards/grading-dataset.js"; import { verifyGoogleToken, generateJwt, verifyJwt } from "./lib/auth/auth.js"; import { seedFromTCGPlayer } from "./lib/sources/tcgplayer.js"; import { getOrCreateCard, findCardByQuery, parseCardIdentity, resolveCardIdToQuery, SET_NAME_MAP } from "./lib/cards/card-identity.js"; -import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js"; +import { initCardDatabase, searchCards, syncCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js"; import { raspMiddleware, getSecurityEvents } from "./lib/security/rasp.js"; import { fileURLToPath } from "url"; import path from "path"; @@ -905,6 +905,18 @@ app.get("/api/sets/:setCode", requireCardDb, (req, res) => { res.json(result); }); +// POST /api/card-database/sync +app.post("/api/card-database/sync", apiAuthMiddleware, async (req, res) => { + if (req.authTier !== "owner") return res.status(403).json({ error: "Owner only" }); + try { + const force = req.query.force === "true"; + const result = await syncCardDatabase({ force }); + res.json(result); + } catch (err) { + res.status(500).json({ error: "Sync failed" }); + } +}); + // ============ V1 API — Drop Intelligence ============ async function authMiddleware(req, res, next) { @@ -2227,7 +2239,7 @@ app.post("/api/track-prices", authMiddleware, async (req, res) => { } catch {} } - refreshCardDatabase().catch(() => {}); + syncCardDatabase().catch(() => {}); res.json({ tracked: results.length, results, portfolioSnapshots, portfolioWarmed, portfolioCardsTracked: portfolioQueries.length, frequencyWarmed }); }); diff --git a/lib/cards/card-database.js b/lib/cards/card-database.js index 9a7f73d..390f9d1 100644 --- a/lib/cards/card-database.js +++ b/lib/cards/card-database.js @@ -168,7 +168,6 @@ function buildImageUrl(image) { } const CACHE_COLLECTION = "card-database-cache"; -const CACHE_TTL_MS = 24 * 60 * 60 * 1000; const CHUNK_SIZE = 5000; function getCacheDb() { @@ -198,9 +197,11 @@ export async function saveCardDatabaseToCache(index) { updatedAt: Date.now(), }); } + const setCodes = [...new Set(index.map(c => c.setCode).filter(Boolean))].sort(); batch.set(fs.collection(CACHE_COLLECTION).doc("meta"), { totalCards: index.length, totalChunks: chunks.length, + setCodes, updatedAt: Date.now(), }); await batch.commit(); @@ -214,7 +215,6 @@ export async function loadCardDatabaseFromCache() { const metaDoc = await fs.collection(CACHE_COLLECTION).doc("meta").get(); if (!metaDoc.exists) return null; const meta = metaDoc.data(); - if (Date.now() - meta.updatedAt > CACHE_TTL_MS) return null; const chunks = meta.totalChunks; const docs = []; @@ -241,19 +241,28 @@ export async function loadCardDatabaseFromCache() { } } +function normalizeCardId(id) { + const dash = id.indexOf("-"); + if (dash === -1) return id; + const prefix = id.slice(0, dash); + const suffix = id.slice(dash + 1).replace(/^0+/, "") || "0"; + return `${prefix}-${suffix}`; +} + function buildIndexFromApi(enCards, jaCards, rarityMap = new Map()) { const jaMap = new Map(); for (const card of jaCards) { - jaMap.set(card.id, card); + jaMap.set(normalizeCardId(card.id), card); } const seen = new Set(); const index = []; for (const card of enCards) { - seen.add(card.id); + const normId = normalizeCardId(card.id); + seen.add(normId); const setCode = parseSetCode(card.id); - const jaCard = jaMap.get(card.id); + const jaCard = jaMap.get(normId); index.push({ id: card.id, name: card.name, @@ -267,7 +276,7 @@ function buildIndexFromApi(enCards, jaCards, rarityMap = new Map()) { } for (const card of jaCards) { - if (seen.has(card.id)) continue; + if (seen.has(normalizeCardId(card.id))) continue; const setCode = parseSetCode(card.id); const enName = translateJaName(card.name); index.push({ @@ -309,18 +318,9 @@ export async function initCardDatabase() { if (cached && cached.length > 0) { cardIndex = cached; deriveSetTotals(cardIndex); + await setMetaPromise; cardDbReady = true; console.log(`Card database loaded from cache: ${cardIndex.length} cards`); - - fetchAndBuildIndex().then(async (freshIndex) => { - if (freshIndex && freshIndex.length > 0) { - cardIndex = freshIndex; - deriveSetTotals(cardIndex); - console.log(`Card database refreshed from TCGdex: ${cardIndex.length} cards`); - await saveCardDatabaseToCache(freshIndex); - } - }).catch(() => {}); - return cardIndex.length; } @@ -328,6 +328,7 @@ export async function initCardDatabase() { if (freshIndex && freshIndex.length > 0) { cardIndex = freshIndex; deriveSetTotals(cardIndex); + await setMetaPromise; cardDbReady = true; console.log(`Card database loaded from TCGdex: ${cardIndex.length} cards`); saveCardDatabaseToCache(freshIndex).catch(() => {}); @@ -342,15 +343,66 @@ export function isCardDatabaseReady() { return cardDbReady; } -export async function refreshCardDatabase() { +async function getCachedSetCodes() { + const fs = getCacheDb(); + if (!fs) return []; + try { + const metaDoc = await fs.collection(CACHE_COLLECTION).doc("meta").get(); + if (!metaDoc.exists) return []; + return metaDoc.data().setCodes || []; + } catch { + return []; + } +} + +async function getTcgdexSetCodes() { + const sets = new Set(); + for (const lang of ["en", "ja"]) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + const res = await fetch(`https://api.tcgdex.net/v2/${lang}/sets`, { signal: controller.signal }); + clearTimeout(timer); + if (!res.ok) continue; + const data = await res.json(); + for (const s of data) { + if (s.id) sets.add(s.id.toLowerCase()); + } + } catch {} + } + return [...sets]; +} + +export async function syncCardDatabase({ force = false } = {}) { + const [cachedCodes, remoteCodes] = await Promise.all([ + getCachedSetCodes(), + getTcgdexSetCodes(), + ]); + + const cachedSet = new Set(cachedCodes.map(c => c.toLowerCase())); + const newSets = remoteCodes.filter(c => !cachedSet.has(c)); + + if (!force && newSets.length === 0) { + return { synced: false, reason: "no new sets", cachedSets: cachedCodes.length, remoteSets: remoteCodes.length }; + } + const freshIndex = await fetchAndBuildIndex(); - if (freshIndex && freshIndex.length > 0) { - cardIndex = freshIndex; - console.log(`Card database refreshed from TCGdex: ${cardIndex.length} cards`); - saveCardDatabaseToCache(freshIndex).catch(() => {}); - return cardIndex.length; + if (!freshIndex || freshIndex.length === 0) { + return { synced: false, reason: "TCGdex fetch failed" }; } - return cardIndex.length; + + cardIndex = freshIndex; + deriveSetTotals(cardIndex); + await saveCardDatabaseToCache(freshIndex); + console.log(`Card database synced: ${cardIndex.length} cards, ${newSets.length} new sets`); + + return { + synced: true, + totalCards: cardIndex.length, + newSets, + cachedSets: cachedCodes.length, + remoteSets: remoteCodes.length, + }; } export function getCardCount() { @@ -434,8 +486,9 @@ export function getAllSets() { } const entry = setMap.get(code); entry.count++; - if (card.lang === "en") entry.enCount++; - else if (card.lang === "jp") entry.jpCount++; + const lang = card.lang || "en"; + if (lang === "en") entry.enCount++; + else if (lang === "jp") entry.jpCount++; if (card.imageUrl) { const num = parseInt(card.localId, 10) || 0; if (num > entry.bestNum) { diff --git a/test/api-test.js b/test/api-test.js index e845b70..d0098e7 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -947,6 +947,11 @@ async function run() { } }); + await test("POST /api/card-database/sync requires owner key", async () => { + const { res } = await json("/api/card-database/sync", { method: "POST" }); + assert(res.status === 403, `expected 403, got ${res.status}`); + }); + // ── Price trend ── console.log("\n\x1b[1m=== price trend ===\x1b[0m");