diff --git a/api.js b/api.js index 214bb53..751b5e9 100644 --- a/api.js +++ b/api.js @@ -15,7 +15,7 @@ import { gradeImage, medianGrade } from "./lib/grading/grading.js"; import { parseListingLanguagesFromInput, filterByCondition, detectCondition, flagPriceOutliers, filterRelevantResults, isGradedCard } from "./lib/search/filters.js"; import { buildEbaySearchQuery } from "./lib/search/listingQuery.js"; import { EBAY_CATEGORY_TCG_SINGLE_CARDS_US } from "./lib/search/ebayCategories.js"; -import { saveGradeLog, getGradeLogs, getGradeLogsByUser, getGradeLog, deleteGradeLog, saveDrop, getDrops, getDrop, saveWebhook, getWebhooks, deleteWebhook, getFirestoreStatus, saveAlert, getActiveAlerts, updateAlert, getAlertsByEmail, saveErrorLog, getErrorLogs, clearErrorLogs, getPortfolio, addToPortfolio, removeFromPortfolio, updatePortfolioCard, savePortfolioSnapshot, getPortfolioSnapshots, listPortfolioUserIds, trackSearchFrequency, getTopSearchedCards } from "./lib/data/firestore.js"; +import { saveGradeLog, getGradeLogs, getGradeLogsByUser, getGradeLog, deleteGradeLog, saveDrop, getDrops, getDrop, saveWebhook, getWebhooks, deleteWebhook, getFirestoreStatus, saveAlert, getActiveAlerts, updateAlert, getAlertsByEmail, saveErrorLog, getErrorLogs, clearErrorLogs, getPortfolio, addToPortfolio, removeFromPortfolio, updatePortfolioCard, savePortfolioSnapshot, getPortfolioSnapshots, listPortfolioUserIds, trackSearchFrequency, getTopSearchedCards, recordMilestone, getFunnelStats } from "./lib/data/firestore.js"; import { getDemoSearchResult, getDemoResult, listDemoCards, findDemoByNumber } from "./lib/cards/demo.js"; import { csvEscape, csvRow } from "./lib/data/csv.js"; import { createApiKey, listApiKeys, listAllKeys, listKeysByOwner, getApiKey, updateApiKey, deleteApiKey, rotateApiKey, validateApiKey } from "./lib/auth/api-keys.js"; @@ -26,7 +26,7 @@ import { saveGradedImages } from "./lib/cards/grading-dataset.js"; import { verifyGoogleToken, generateJwt, verifyJwt } from "./lib/auth/auth.js"; import { seedFromTCGPlayer } from "./lib/sources/tcgplayer.js"; import { getOrCreateCard, findCardByQuery, parseCardIdentity, resolveCardIdToQuery, SET_NAME_MAP } from "./lib/cards/card-identity.js"; -import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId } from "./lib/cards/card-database.js"; +import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js"; import { raspMiddleware, getSecurityEvents } from "./lib/security/rasp.js"; import { fileURLToPath } from "url"; import path from "path"; @@ -184,6 +184,11 @@ const clientSecret = process.env.EBAY_CLIENT_SECRET; async function getToken() { return getAccessToken(clientId, clientSecret); } async function on401() { invalidateToken(); } +function requireCardDb(req, res, next) { + if (!isCardDatabaseReady()) return res.status(503).json({ error: "Card database loading, try again shortly", retryAfter: 5 }); + next(); +} + function validateQuery(q, res) { if (!q) { res.status(400).json({ error: "Missing required parameter: q" }); return false; } if (q.length > 200) { res.status(400).json({ error: "Query too long (max 200 characters)" }); return false; } @@ -372,6 +377,7 @@ app.get("/api/search", apiAuthMiddleware, (req, res, next) => { req._errorType = } getOrCreateCard(q, { source: result.source, lang: config.language }).catch(() => {}); trackSearchFrequency(q).catch(() => {}); + if (req.userId) recordMilestone(req.userId, "firstSearch").catch(() => {}); res.json(result); } catch (e) { logError(req._errorType || "api", e.message, req.originalUrl, req.requestId); @@ -503,6 +509,7 @@ app.post("/api/grade", authMiddleware, (req, res, next) => { req._errorType = "g } } + if (req.userId && grade && !grade.error) recordMilestone(req.userId, "firstGrade").catch(() => {}); res.json({ grade, gradeId, stored: !!(grade && !grade.error) }); } catch (e) { logError(req._errorType || "api", e.message, req.originalUrl, req.requestId); @@ -613,8 +620,9 @@ app.get("/api/grades", authMiddleware, async (req, res) => { // GET /api/errors app.get("/api/errors", authMiddleware, async (req, res) => { const limit = Math.min(100, Math.max(1, Number(req.query.limit) || 20)); + const type = req.query.type || undefined; try { - const errors = await getErrorLogs({ limit }); + const errors = await getErrorLogs({ limit, type }); res.json({ errors, count: errors.length }); } catch (e) { res.status(500).json({ error: safeErrorMessage(e), requestId: req.requestId }); @@ -642,6 +650,16 @@ app.get("/api/analytics", ownerOnly, async (req, res) => { } }); +// GET /api/funnel +app.get("/api/funnel", ownerOnly, async (req, res) => { + try { + const stats = await getFunnelStats(); + res.json(stats); + } catch (e) { + res.status(500).json({ error: safeErrorMessage(e), requestId: req.requestId }); + } +}); + // GET /api/grading-dataset/stats app.get("/api/grading-dataset/stats", ownerOnly, async (req, res) => { try { @@ -708,6 +726,7 @@ app.post("/auth/google", authLimiter, async (req, res) => { } const isAdmin = gUser.sub === process.env.CASECOMP_ADMIN_SUB; + recordMilestone(gUser.sub, "signup").catch(() => {}); res.json({ jwt, apiKey, isAdmin, user: { id: gUser.sub, email: gUser.email, name: gUser.name, picture: gUser.picture } }); } catch (e) { res.status(401).json({ error: "Invalid Google token" }); @@ -860,7 +879,7 @@ app.delete("/api/admin/keys/:id", authMiddleware, async (req, res) => { }); // GET /api/autocomplete -app.get("/api/autocomplete", (req, res) => { +app.get("/api/autocomplete", requireCardDb, (req, res) => { const q = (req.query.q || "").trim(); if (!q || q.length < 2) return res.status(400).json({ error: "Query must be at least 2 characters" }); if (q.length > 100) return res.status(400).json({ error: "Query too long (max 100 characters)" }); @@ -870,7 +889,7 @@ app.get("/api/autocomplete", (req, res) => { }); // GET /api/sets -app.get("/api/sets", (req, res) => { +app.get("/api/sets", requireCardDb, (req, res) => { const sets = getAllSets(); const era = req.query.era; const filtered = era ? sets.filter(s => s.era === era) : sets; @@ -878,7 +897,7 @@ app.get("/api/sets", (req, res) => { }); // GET /api/sets/:setCode -app.get("/api/sets/:setCode", (req, res) => { +app.get("/api/sets/:setCode", requireCardDb, (req, res) => { const result = getSetWithCards(req.params.setCode); if (!result) return res.status(404).json({ error: "Set not found" }); res.json(result); @@ -1999,6 +2018,7 @@ app.post("/api/portfolio", authMiddleware, async (req, res) => { purchaseSource: purchaseSource || "", quantity: quantity != null ? Number(quantity) : 1, }); + recordMilestone(userId, "firstPortfolioAdd").catch(() => {}); res.status(201).json({ ok: true, card }); } catch (e) { logError("portfolio", e.message, req.originalUrl, req.requestId); @@ -2413,9 +2433,8 @@ app.use((err, req, res, _next) => { }); const PORT = process.env.API_PORT || 3000; -app.listen(PORT, async () => { - console.log(`Casecomp API listening on http://localhost:${PORT}`); - console.log(`Swagger docs: http://localhost:${PORT}/docs`); + +async function startup() { if (clientId && clientSecret) { try { await getToken(); @@ -2424,5 +2443,25 @@ app.listen(PORT, async () => { console.warn(`eBay token warmup failed: ${e.message}`); } } - initCardDatabase().catch(() => {}); + try { + await initCardDatabase(); + } catch (e) { + console.warn(`Card database init failed: ${e.message}`); + } + app.listen(PORT, () => { + console.log(`Casecomp API listening on http://localhost:${PORT}`); + console.log(`Swagger docs: http://localhost:${PORT}/docs`); + }); +} + +startup(); + +process.on("unhandledRejection", (reason) => { + const msg = reason instanceof Error ? reason.message : String(reason); + logError("unhandledRejection", msg, reason instanceof Error ? reason.stack?.split("\n")[1]?.trim() : ""); +}); + +process.on("uncaughtException", (err) => { + logError("uncaughtException", err.message, err.stack?.split("\n")[1]?.trim()); + setTimeout(() => process.exit(1), 1000); }); diff --git a/lib/cards/card-database.js b/lib/cards/card-database.js index bfed098..39b045d 100644 --- a/lib/cards/card-database.js +++ b/lib/cards/card-database.js @@ -60,6 +60,7 @@ const RARITY_MAP = { const RARITY_TIERS = Object.keys(RARITY_MAP); let cardIndex = []; +let cardDbReady = false; const tcgdexSetMeta = new Map(); async function fetchSetMetadata(timeout = 15000) { @@ -306,6 +307,7 @@ export async function initCardDatabase() { if (cached && cached.length > 0) { cardIndex = cached; deriveSetTotals(cardIndex); + cardDbReady = true; console.log(`Card database loaded from cache: ${cardIndex.length} cards`); fetchAndBuildIndex().then(async (freshIndex) => { @@ -324,6 +326,7 @@ export async function initCardDatabase() { if (freshIndex && freshIndex.length > 0) { cardIndex = freshIndex; deriveSetTotals(cardIndex); + cardDbReady = true; console.log(`Card database loaded from TCGdex: ${cardIndex.length} cards`); saveCardDatabaseToCache(freshIndex).catch(() => {}); return cardIndex.length; @@ -333,6 +336,10 @@ export async function initCardDatabase() { return 0; } +export function isCardDatabaseReady() { + return cardDbReady; +} + export async function refreshCardDatabase() { const freshIndex = await fetchAndBuildIndex(); if (freshIndex && freshIndex.length > 0) { diff --git a/lib/data/analytics.js b/lib/data/analytics.js index 8d07507..82cdd55 100644 --- a/lib/data/analytics.js +++ b/lib/data/analytics.js @@ -68,14 +68,26 @@ export async function getAnalytics({ days = 7, limit = 1000 } = {}) { const byTier = {}; const byPath = {}; + const byStatus = {}; const queries = {}; + const dailyMap = {}; + const users = new Set(); let totalLatency = 0; for (const d of docs) { byTier[d.tier] = (byTier[d.tier] || 0) + 1; byPath[d.path] = (byPath[d.path] || 0) + 1; + const sc = d.status ? String(d.status).charAt(0) + "xx" : "unknown"; + byStatus[sc] = (byStatus[sc] || 0) + 1; if (d.query) queries[d.query] = (queries[d.query] || 0) + 1; + if (d.userId) users.add(d.userId); totalLatency += d.latencyMs || 0; + const day = d.ts?.split("T")[0]; + if (day) { + if (!dailyMap[day]) dailyMap[day] = { count: 0, latency: 0 }; + dailyMap[day].count++; + dailyMap[day].latency += d.latencyMs || 0; + } } const topQueries = Object.entries(queries) @@ -83,13 +95,20 @@ export async function getAnalytics({ days = 7, limit = 1000 } = {}) { .slice(0, 10) .map(([query, count]) => ({ query, count })); + const daily = Object.entries(dailyMap) + .sort() + .map(([date, v]) => ({ date, count: v.count, avgLatencyMs: v.count > 0 ? Math.round(v.latency / v.count) : 0 })); + return { total, days, byTier, byPath, + byStatus, topQueries, avgLatencyMs: total > 0 ? Math.round(totalLatency / total) : 0, + uniqueUsers: users.size, + daily, }; } catch { return { total: 0, byTier: {}, byPath: {}, topQueries: [], avgLatencyMs: 0 }; diff --git a/lib/data/firestore.js b/lib/data/firestore.js index 78ddea0..bfc2d6c 100644 --- a/lib/data/firestore.js +++ b/lib/data/firestore.js @@ -265,6 +265,7 @@ export async function saveErrorLog(record) { await fs.collection("error-logs").add({ ...record, createdAt: Firestore.FieldValue.serverTimestamp(), + expiresAt: new Date(Date.now() + 30 * 86400000), }); } catch {} } @@ -281,17 +282,63 @@ export async function clearErrorLogs() { } catch { return 0; } } -export async function getErrorLogs({ limit = 20 } = {}) { +export async function getErrorLogs({ limit = 20, type } = {}) { const fs = getDb(); if (!fs) return []; try { - const snap = await fs.collection("error-logs").orderBy("createdAt", "desc").limit(limit).get(); + let q = fs.collection("error-logs"); + if (type) q = q.where("type", "==", type); + const snap = await q.orderBy("createdAt", "desc").limit(limit).get(); return snap.docs.map(d => ({ id: d.id, ...d.data() })); } catch { return []; } } +// ── User milestones (funnel tracking) ── + +const MILESTONES_COLLECTION = "user-milestones"; +const MILESTONE_FIELDS = ["signup", "firstSearch", "firstGrade", "firstPortfolioAdd"]; + +export async function recordMilestone(userId, field) { + if (!userId || !MILESTONE_FIELDS.includes(field)) return; + const fs = getDb(); + if (!fs) return; + try { + const ref = fs.collection(MILESTONES_COLLECTION).doc(userId); + const doc = await ref.get(); + if (doc.exists && doc.data()[field]) return; + await ref.set( + { userId, [field]: new Date().toISOString(), updatedAt: new Date().toISOString() }, + { merge: true }, + ); + } catch {} +} + +let funnelCache = null; +let funnelCacheAt = 0; + +export async function getFunnelStats() { + if (funnelCache && Date.now() - funnelCacheAt < 300000) return funnelCache; + const fs = getDb(); + if (!fs) return { signup: 0, firstSearch: 0, firstGrade: 0, firstPortfolioAdd: 0 }; + try { + const snap = await fs.collection(MILESTONES_COLLECTION).limit(1000).get(); + const result = { signup: 0, firstSearch: 0, firstGrade: 0, firstPortfolioAdd: 0 }; + for (const doc of snap.docs) { + const d = doc.data(); + for (const f of MILESTONE_FIELDS) { + if (d[f]) result[f]++; + } + } + funnelCache = result; + funnelCacheAt = Date.now(); + return result; + } catch { + return { signup: 0, firstSearch: 0, firstGrade: 0, firstPortfolioAdd: 0 }; + } +} + // ── Cache (replaces file-based JSON caches) ── export async function cacheGet(collection, key) { diff --git a/public/admin/admin.css b/public/admin/admin.css index 24bd796..372a4df 100644 --- a/public/admin/admin.css +++ b/public/admin/admin.css @@ -90,6 +90,20 @@ main { justify-content: space-between; margin-bottom: 16px; } +.toolbar-right { display: flex; gap: 8px; align-items: center; } + +select { + padding: 6px 10px; + background: var(--inset); + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + font-size: 12px; + font-family: inherit; + outline: none; + cursor: pointer; +} +select:focus { border-color: rgba(217, 182, 118, 0.4); } .card { background: var(--panel); @@ -264,4 +278,89 @@ main { font-size: 13px; } +.tab-bar { + display: flex; + gap: 0; + border-bottom: 1px solid var(--border); + margin-bottom: 20px; +} +.tab { + background: none; + border: none; + border-bottom: 2px solid transparent; + color: var(--muted); + padding: 10px 20px; + font-family: 'Space Grotesk', system-ui, sans-serif; + font-size: 13px; + font-weight: 600; + cursor: pointer; + border-radius: 0; +} +.tab:hover { color: var(--text); } +.tab.active { color: var(--gold); border-bottom-color: var(--gold); } + +.analytics-columns { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; } + +.bar-chart { + display: flex; + align-items: flex-end; + gap: 3px; + height: 140px; + padding: 8px 0; +} +.bar-col { display: flex; flex-direction: column; align-items: center; flex: 1; min-width: 0; } +.bar { + width: 100%; + background: var(--gold); + border-radius: 3px 3px 0 0; + min-height: 2px; +} +.bar-date { + font-size: 9px; + color: var(--muted); + margin-top: 4px; + white-space: nowrap; +} + +.hbar-row { display: flex; align-items: center; gap: 8px; padding: 6px 0; } +.hbar-label { + font-size: 12px; + color: var(--text); + min-width: 100px; + font-family: monospace; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.hbar-track { flex: 1; height: 18px; background: var(--inset); border-radius: 4px; overflow: hidden; } +.hbar-fill { height: 100%; background: var(--gold); border-radius: 4px; transition: width 0.3s; } +.hbar-count { font-size: 12px; color: var(--muted); min-width: 40px; text-align: right; font-family: monospace; } + +.query-table { width: 100%; border-collapse: collapse; font-size: 12px; } +.query-table th { + text-align: left; + color: var(--muted); + font-weight: 500; + padding: 6px 8px; + border-bottom: 1px solid var(--border); + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.06em; +} +.query-table td { padding: 6px 8px; border-bottom: 1px solid var(--border); } +.query-table td:last-child { text-align: right; color: var(--muted); font-family: monospace; } + +.funnel-row { display: flex; align-items: center; gap: 12px; padding: 12px 0; } +.funnel-label { font-size: 13px; color: var(--text); min-width: 140px; font-weight: 500; } +.funnel-track { flex: 1; height: 28px; background: var(--inset); border-radius: 6px; overflow: hidden; } +.funnel-fill { height: 100%; background: var(--gold); border-radius: 6px; } +.funnel-count { + font-family: 'Space Grotesk', system-ui, sans-serif; + font-size: 18px; + font-weight: 700; + min-width: 40px; + text-align: right; +} +.funnel-pct { font-size: 12px; color: var(--muted); min-width: 40px; } + .hidden { display: none !important; } diff --git a/public/admin/admin.js b/public/admin/admin.js index 07597f6..9d3b13f 100644 --- a/public/admin/admin.js +++ b/public/admin/admin.js @@ -218,7 +218,9 @@ async function deleteKey(id) { async function loadErrors() { try { - const res = await fetch(`/api/errors?limit=20`, { headers: { Authorization: `Bearer ${ownerKey}` } }); + const typeFilter = document.getElementById("error-type-filter")?.value || ""; + const typeParam = typeFilter ? `&type=${encodeURIComponent(typeFilter)}` : ""; + const res = await fetch(`/api/errors?limit=20${typeParam}`, { headers: { Authorization: `Bearer ${ownerKey}` } }); const { errors } = await res.json(); if (!errors || !errors.length) { $("#error-list").innerHTML = '
No errors
'; @@ -244,3 +246,106 @@ function esc(s) { d.textContent = String(s); return d.innerHTML; } + +document.querySelectorAll(".tab").forEach(tab => { + tab.addEventListener("click", () => { + document.querySelectorAll(".tab").forEach(t => t.classList.remove("active")); + document.querySelectorAll(".tab-content").forEach(c => c.classList.add("hidden")); + tab.classList.add("active"); + const target = document.getElementById(`tab-${tab.dataset.tab}`); + if (target) target.classList.remove("hidden"); + if (tab.dataset.tab === "analytics") loadAnalytics(); + if (tab.dataset.tab === "funnel") loadFunnel(); + }); +}); + +function hBar(items, maxVal) { + if (!items.length) return '
No data
'; + return items.map(([label, count]) => { + const pct = maxVal > 0 ? Math.round((count / maxVal) * 100) : 0; + return `
+ ${esc(label)} +
+ ${count.toLocaleString()} +
`; + }).join(""); +} + +async function loadAnalytics() { + const days = parseInt(document.getElementById("analytics-days")?.value) || 7; + try { + const res = await fetch(`/api/analytics?days=${days}`, { headers: { Authorization: `Bearer ${ownerKey}` } }); + const data = await res.json(); + + const latClass = data.avgLatencyMs < 200 ? "ok" : data.avgLatencyMs < 500 ? "warn" : "bad"; + $("#analytics-stats").innerHTML = ` +
Total Requests
${data.total.toLocaleString()}
+
Unique Users
${(data.uniqueUsers || 0).toLocaleString()}
+
Avg Latency
${data.avgLatencyMs}ms
+ `; + + const daily = data.daily || []; + if (daily.length) { + const maxCount = Math.max(...daily.map(d => d.count)); + $("#analytics-daily").innerHTML = `
${daily.map(d => { + const h = maxCount > 0 ? Math.max(2, Math.round((d.count / maxCount) * 120)) : 2; + const label = d.date.slice(5); + return `
${label}
`; + }).join("")}
`; + } else { + $("#analytics-daily").innerHTML = '
No data yet
'; + } + + const tierEntries = Object.entries(data.byTier || {}).sort((a, b) => b[1] - a[1]); + const tierMax = tierEntries.length ? tierEntries[0][1] : 0; + $("#analytics-tier").innerHTML = hBar(tierEntries, tierMax); + + const statusEntries = Object.entries(data.byStatus || {}).sort((a, b) => b[1] - a[1]); + const statusMax = statusEntries.length ? statusEntries[0][1] : 0; + $("#analytics-status").innerHTML = hBar(statusEntries, statusMax); + + const pathEntries = Object.entries(data.byPath || {}).sort((a, b) => b[1] - a[1]).slice(0, 10); + const pathMax = pathEntries.length ? pathEntries[0][1] : 0; + $("#analytics-paths").innerHTML = hBar(pathEntries, pathMax); + + const queries = data.topQueries || []; + if (queries.length) { + $("#analytics-queries").innerHTML = ` + + ${queries.map(q => ``).join("")} +
QueryCount
${esc(q.query)}${q.count}
`; + } else { + $("#analytics-queries").innerHTML = '
No queries yet
'; + } + } catch (e) { + $("#analytics-stats").innerHTML = `
${esc(e.message)}
`; + } +} + +async function loadFunnel() { + try { + const res = await fetch("/api/funnel", { headers: { Authorization: `Bearer ${ownerKey}` } }); + const data = await res.json(); + const stages = [ + { label: "Signups", key: "signup" }, + { label: "First Search", key: "firstSearch" }, + { label: "First Grade", key: "firstGrade" }, + { label: "First Portfolio Add", key: "firstPortfolioAdd" }, + ]; + const maxVal = data.signup || 1; + $("#funnel-chart").innerHTML = stages.map((s, i) => { + const count = data[s.key] || 0; + const pct = Math.round((count / maxVal) * 100); + const convLabel = i > 0 ? `${pct}%` : ""; + const opacity = 1 - i * 0.15; + return `
+ ${s.label} +
+ ${count} + ${convLabel} +
`; + }).join(""); + } catch (e) { + $("#funnel-chart").innerHTML = `
${esc(e.message)}
`; + } +} diff --git a/public/admin/index.html b/public/admin/index.html index b3fa454..7e3eb57 100644 --- a/public/admin/index.html +++ b/public/admin/index.html @@ -26,24 +26,84 @@ API Docs -
-

Developer Keys

- +
+ + + +
-