diff --git a/.gitignore b/.gitignore index 186248b..69a4e10 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ github.token sync*.log .brain-state.json +.brain-ledger.json +.brain-viewer.json # 飞书 PoC 凭证 feishu-poc/secret.yaml diff --git a/cli/brain.mjs b/cli/brain.mjs index c928968..ebcde27 100644 --- a/cli/brain.mjs +++ b/cli/brain.mjs @@ -470,8 +470,21 @@ function status() { const last = readFileSync(LOG, "utf8").trimEnd().split("\n").filter((l) => l.includes("·")).slice(-1)[0]; if (last) console.log(`最近同步: ${c.dim(last)}`); } + try { const vi = JSON.parse(readFileSync(join(ROOT, ".brain-viewer.json"), "utf8")); if (vi.url) console.log(`查看器: ${c.ok(vi.url)} ${c.dim("(brain viewer 打开)")}`); } catch {} if (!running && existsSync(CFG)) console.log(c.dim("\n起常驻：brain service install")); } + +// ---------------- viewer（本机足迹查看器）---------------- +// 常驻 sync 内嵌了一个 127.0.0.1 只读小服务（client/viewer.mjs），把地址写进 .brain-viewer.json。 +// 这里读出来用浏览器打开。没在跑就提示先起常驻。 +function viewer() { + let info; try { info = JSON.parse(readFileSync(join(ROOT, ".brain-viewer.json"), "utf8")); } catch {} + if (!info?.url) die("本机查看器还没起（需要常驻在跑）。先 brain service install（或 brain start），再 brain viewer。"); + console.log(c.ok("本机足迹查看器：") + " " + info.url + c.dim(" （仅本机可见）")); + const opener = IS_MAC ? "open" : "xdg-open"; + if (has(opener)) sh(opener, [info.url]); + else console.log(c.dim("（手动在浏览器打开上面的地址）")); +} function logs(follow) { if (!existsSync(LOG)) die("还没有日志（没跑过 sync）"); const args = follow ? ["-n", "40", "-f", LOG] : ["-n", "40", LOG]; @@ -495,6 +508,7 @@ const HELP = `brain —— 团队大脑客户端 brain stop 停常驻（不卸） brain uninstall [--purge] 完整卸载（停常驻+摘MCP+删token配置；--purge 连命令和安装目录） brain status 看状态 + brain viewer 打开本机足迹查看器（127.0.0.1，仅本机可见） brain version 看客户端版本 brain logs [-f] 看日志 brain admin add --name <显示名> [--email][--git-name][--consumer] （管理员）加人，吐邀请码 @@ -526,6 +540,7 @@ try { case "restart": serviceRestart(); break; case "stop": serviceStop(); break; case "status": case undefined: status(); break; + case "viewer": viewer(); break; // 打开本机足迹查看器（常驻内嵌的 127.0.0.1 服务） case "logs": logs(flag("-f") || flag("--follow")); break; case "version": case "-v": case "--version": console.log(`brain ${CLIENT_VERSION}`); break; case "help": case "-h": case "--help": console.log(HELP); break; diff --git a/client/sync.mjs b/client/sync.mjs index b95d7c1..a3534e2 100644 --- a/client/sync.mjs +++ b/client/sync.mjs @@ -14,6 +14,10 @@ import { parseSession, parseSessionText } from "../core/parse.mjs"; import { coordOf, expandHome, gitBranch, parseRemote } from "../core/coord.mjs"; import { slimRaw, slimRawFile } from "../core/slim.mjs"; import { log } from "../core/log.mjs"; +import { loadLedger, saveLedger, recordSession, getByFile } from "../core/ledger.mjs"; +import { loadOptout, isOptedOut } from "../core/optout.mjs"; +import { loadUserRedact, applyUserRedact } from "../core/userredact.mjs"; +import { startViewer } from "./viewer.mjs"; import { CLIENT_VERSION, PIPELINE_VERSION } from "../core/version.mjs"; const MAX_UPLOAD = 80 * 1024 * 1024; // 蒸馏后仍超此值 → 跳过+标 seen。放到 80MB：超活跃 session（数千次 @@ -37,6 +41,10 @@ const SESSION_HISTORY_DIR = "session_history"; const DEBOUNCE = (cfg.debounce_sec ?? 60) * 1000; const CONC = Math.max(1, cfg.concurrency ?? 4); // 上传并发：一个卡住的不再串行阻塞整轮 const STATE = join(ROOT, ".brain-state.json"); // 已上传记录持久化：重启不再重传历史 +const LEDGER = join(ROOT, ".brain-ledger.json"); // 结果账本：viewer 的数据源（传了啥/没传啥/为什么/在库哪） +const OPTOUT = join(ROOT, ".brain-optout.json"); // 逐条排除名单：命中即永不上传 +const USERREDACT = join(ROOT, ".brain-redact.json"); // 个人脱敏词表：上传前额外抹除 +const BACKFILL_PARSE_CAP = 12 * 1024 * 1024; // 历史回填：超此大小不细解，按大小/位置粗判，避免回填卡顿 const AUTO_UPDATE = cfg.auto_update !== false; // 自动更新：默认开；client.config.yaml 写 auto_update:false 可关 const UPDATE_CHECK_MS = 24 * 3600 * 1000; // 每天最多自检一次（够用、不打扰；部署后最长一天全队收敛） const IS_DEV = existsSync(join(ROOT, ".git")); // 开发 checkout 不自动更新（applyUpdate 也会拦，这里先省一次往返） @@ -108,6 +116,14 @@ const gated = (cwd) => const traeSlug = (p) => expandHome(p).replace(/[^A-Za-z0-9]/g, "-"); const traeMemoryProjectsConfig = () => cfg.trae_memory_projects || cfg.trae_memory_folders || []; +// 账本字段：从解析结果 s + 坐标 c 摘出 viewer 要显示的便宜元信息（坐标/意图/工作时间/turns）。 +const coordFields = (s, c, branch) => ({ + intent: s.intent, cwd: s.cwd, remote: c?.remote || null, folder: c?.folder || null, + branch: branch ?? s.branch, work_start: s.ts, work_end: s.updated, turns: s.turns, +}); +// 从 /ingest 200 响应体取服务端定下的规范坐标（space_key/file），记进账本好让 viewer 显示「在库哪」。 +const serverCoord = (r) => { try { return JSON.parse(r.body) || {}; } catch { return {}; } }; + async function post(path, payload, tries = 3) { const json = JSON.stringify(payload); const body = gzipSync(Buffer.from(json)); // gzip：跨境省 ~10x 流量 @@ -154,23 +170,30 @@ async function syncSessions() { const m = st.mtimeMs; if (Date.now() - m < DEBOUNCE) continue; // 还在活跃，等稳定 if (seenSession.get(file) === m) continue; // 没变 - if (st.size > MAX_RAW) { log.warn("ingest 跳过：原文超读前上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; continue; } + const idEarly = basename(file).replace(/\.jsonl$/, ""); + if (isOptedOut(idEarly, file)) { seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idEarly, tool: "claude-code", status: "opted_out" }); continue; } + if (st.size > MAX_RAW) { log.warn("ingest 跳过：原文超读前上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idEarly, tool: "claude-code", status: "skipped", reason: "toobig", bytes_raw: st.size }); continue; } const s = parseSession(file); - if (!s.intent || !gated(s.cwd)) { skip++; seenSession.set(file, m); continue; } - jobs.push({ file, m, s }); + if (!s.intent) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id: idEarly, tool: "claude-code", status: "skipped", reason: "nointent", cwd: s.cwd }); continue; } + if (!gated(s.cwd)) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id: idEarly, tool: "claude-code", status: "skipped", reason: "gated", intent: s.intent, cwd: s.cwd }); continue; } + jobs.push({ file, m, s, size: st.size }); } - await pool(jobs, CONC, async ({ file, m, s }) => { + await pool(jobs, CONC, async ({ file, m, s, size }) => { const id = basename(file).replace(/\.jsonl$/, ""); try { const c = coordOf(s.cwd, cfg.upload_folders); // 只算 remote + folder；github-vs-local 由服务器定 - const raw = slimRaw(readFileSync(file, "utf8")); // 上传前瘦身：剥图片/截巨型输出，完整原文留本机 - if (raw.length > MAX_UPLOAD) { log.warn("ingest 跳过：瘦身后仍超上限", { id, mb: +(raw.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; return; } + const raw = applyUserRedact(slimRaw(readFileSync(file, "utf8"))); // 上传前瘦身 + 个人脱敏词表；完整原文留本机 + if (raw.length > MAX_UPLOAD) { log.warn("ingest 跳过：瘦身后仍超上限", { id, mb: +(raw.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id, tool: "claude-code", status: "skipped", reason: "toobig", ...coordFields(s, c), bytes_raw: size, bytes_slim: raw.length }); return; } const r = await post("/ingest", { id, tool: "claude-code", raw, remote: c.remote, folder: c.folder, branch: s.branch, producer: cfg.me, }); - if (r.status === 200) { up++; seenSession.set(file, m); } else log.warn("ingest 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); + if (r.status === 200) { + up++; seenSession.set(file, m); + const sk = serverCoord(r); + recordSession({ file, mtime: m, id, tool: "claude-code", status: "uploaded", ...coordFields(s, c), space_key: sk.space_key, server_file: sk.file, bytes_raw: size, bytes_slim: raw.length }); + } else log.warn("ingest 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); } catch (e) { log.error("ingest 出错", { id, err: e.message }); } // 读/瘦身/上传任一出错只困住这条，不拖垮整轮；失败不 mark seen → 下一轮重试 }); return { up, skip }; @@ -195,28 +218,36 @@ async function syncCodexSessions() { const m = st.mtimeMs; if (Date.now() - m < DEBOUNCE) continue; // 还在活跃，等稳定 if (seenSession.get(file) === m) continue; // 没变 - if (st.size > MAX_RAW_CODEX) { log.warn("codex 跳过：原文超采集上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; continue; } - jobs.push({ file, m }); // 读/蒸馏/判闸门都挪进 worker：流式逐行，不再整文件读成大字符串 + const idC = basename(file).replace(/\.jsonl$/, ""); + if (isOptedOut(idC, file)) { seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idC, tool: "codex", status: "opted_out" }); continue; } + if (st.size > MAX_RAW_CODEX) { log.warn("codex 跳过：原文超采集上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idC, tool: "codex", status: "skipped", reason: "toobig", bytes_raw: st.size }); continue; } + jobs.push({ file, m, size: st.size }); // 读/蒸馏/判闸门都挪进 worker：流式逐行，不再整文件读成大字符串 } - await pool(jobs, CONC, async ({ file, m }) => { + await pool(jobs, CONC, async ({ file, m, size }) => { const id = basename(file).replace(/\.jsonl$/, ""); // rollout--，唯一 try { const slim = await slimRawFile(file); // 流式蒸馏：逐行读 → 剥图片/截巨型输出，完整原文留本机；几百 MB 也不撞 V8 串上限 const s = parseSessionText(slim, "codex"); // 用蒸馏后文本判闸门/取坐标（messages + session_meta.git 都保留） - // 没人类开场 / 不在闸门内 / guardian-子代理噪声 → 跳过 - if (!s.intent || s.subagent || !gated(s.cwd)) { skip++; seenSession.set(file, m); return; } + // 没人类开场 / guardian-子代理噪声 / 不在闸门内 → 跳过（分别记原因） + if (!s.intent) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id, tool: "codex", status: "skipped", reason: "nointent", cwd: s.cwd }); return; } + if (s.subagent) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id, tool: "codex", status: "skipped", reason: "subagent", intent: s.intent, cwd: s.cwd }); return; } + if (!gated(s.cwd)) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id, tool: "codex", status: "skipped", reason: "gated", intent: s.intent, cwd: s.cwd }); return; } const c = coordOf(s.cwd, cfg.upload_folders); // 只算 remote + folder；github-vs-local 由服务器定 // remote/branch 优先用原文 session_meta.git 记的（session 时刻、可靠）； // 没记（旧 rollout）才回退到现场 git：remote 按 cwd 现取 origin、branch 按 cwd 现取当前分支。 const remote = c.remote || parseRemote(s.repoUrl); const branch = s.branch || gitBranch(s.cwd); - if (slim.length > MAX_UPLOAD) { log.warn("codex 跳过：瘦身后仍超上限", { id, mb: +(slim.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; return; } + if (slim.length > MAX_UPLOAD) { log.warn("codex 跳过：瘦身后仍超上限", { id, mb: +(slim.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id, tool: "codex", status: "skipped", reason: "toobig", ...coordFields(s, { remote, folder: c.folder }, branch), bytes_raw: size, bytes_slim: slim.length }); return; } const r = await post("/ingest", { - id, tool: "codex", raw: slim, + id, tool: "codex", raw: applyUserRedact(slim), remote, folder: c.folder, branch, producer: cfg.me, }); - if (r.status === 200) { up++; seenSession.set(file, m); } else log.warn("codex 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); + if (r.status === 200) { + up++; seenSession.set(file, m); + const sk = serverCoord(r); + recordSession({ file, mtime: m, id, tool: "codex", status: "uploaded", ...coordFields(s, { remote, folder: c.folder }, branch), space_key: sk.space_key, server_file: sk.file, bytes_raw: size, bytes_slim: slim.length }); + } else log.warn("codex 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); } catch (e) { log.error("codex 出错", { id, err: e.message }); } // 单文件出错只困住这条，不拖垮整轮 }); return { up, skip }; @@ -284,11 +315,13 @@ async function syncSessionHistoryMd() { const m = st.mtimeMs; if (Date.now() - m < DEBOUNCE) continue; if (seenSession.get(file) === m) continue; - if (st.size > MAX_RAW) { log.warn("session-history 跳过：原文超读前上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; continue; } + const idH = sessionHistoryId(file); + if (isOptedOut(idH, file)) { seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idH, tool: "session-history-md", status: "opted_out" }); continue; } + if (st.size > MAX_RAW) { log.warn("session-history 跳过：原文超读前上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idH, tool: "session-history-md", status: "skipped", reason: "toobig", bytes_raw: st.size }); continue; } const cwd = sessionHistoryProjectDir(file); - if (!gated(cwd) || !gated(file)) { skip++; seenSession.set(file, m); continue; } + if (!gated(cwd) || !gated(file)) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id: sessionHistoryId(file), tool: "session-history-md", status: "skipped", reason: "gated", cwd }); continue; } let content; try { content = readFileSync(file, "utf8"); } catch (e) { log.warn("session-history 读取失败", { file: basename(file), err: e.message }); continue; } - if (!content.trim()) { skip++; seenSession.set(file, m); continue; } + if (!content.trim()) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id: sessionHistoryId(file), tool: "session-history-md", status: "skipped", reason: "empty", cwd }); continue; } jobs.push({ file, m, cwd, content }); } await pool(jobs, CONC, async ({ file, m, cwd, content }) => { @@ -296,14 +329,19 @@ async function syncSessionHistoryMd() { try { const c = coordOf(cwd, cfg.upload_folders); const branch = gitBranch(cwd); + const s = parseSessionText(content, "session-history-md"); // 取 intent/时间给账本（md 解析很便宜） const raw = sessionHistoryRaw({ file, cwd, branch, content, mtimeMs: m }); - if (raw.length > MAX_UPLOAD) { log.warn("session-history 跳过：上传体超上限", { id, mb: +(raw.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; return; } + if (raw.length > MAX_UPLOAD) { log.warn("session-history 跳过：上传体超上限", { id, mb: +(raw.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id, tool: "session-history-md", status: "skipped", reason: "toobig", ...coordFields({ ...s, cwd }, c, branch), bytes_slim: raw.length }); return; } const r = await post("/ingest", { - id, tool: "session-history-md", raw, + id, tool: "session-history-md", raw: applyUserRedact(raw), remote: c.remote, folder: c.folder, branch, producer: cfg.me, }); - if (r.status === 200) { up++; seenSession.set(file, m); } else log.warn("session-history 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); + if (r.status === 200) { + up++; seenSession.set(file, m); + const sk = serverCoord(r); + recordSession({ file, mtime: m, id, tool: "session-history-md", status: "uploaded", ...coordFields({ ...s, cwd }, c, branch), space_key: sk.space_key, server_file: sk.file, bytes_slim: raw.length }); + } else log.warn("session-history 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); } catch (e) { log.error("session-history 出错", { id, err: e.message }); } }); return { up, skip }; @@ -373,38 +411,82 @@ async function syncTraeSessionMemory() { const m = st.mtimeMs; if (Date.now() - m < DEBOUNCE) continue; if (seenSession.get(file) === m) continue; - if (st.size > MAX_RAW) { log.warn("trae-memory 跳过：原文超读前上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; continue; } + const idT = traeMemoryId(file); + if (isOptedOut(idT, file)) { seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idT, tool: "trae-session-memory", status: "opted_out" }); continue; } + if (st.size > MAX_RAW) { log.warn("trae-memory 跳过：原文超读前上限", { file: basename(file), mb: +(st.size / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id: idT, tool: "trae-session-memory", status: "skipped", reason: "toobig", bytes_raw: st.size }); continue; } let raw; try { raw = readFileSync(file, "utf8"); } catch (e) { log.warn("trae-memory 读取失败", { file: basename(file), err: e.message }); continue; } const s = parseSessionText(raw, "trae-session-memory"); - if (!s.intent) { skip++; seenSession.set(file, m); continue; } - jobs.push({ file, m, project, raw }); + if (!s.intent) { skip++; seenSession.set(file, m); recordSession({ file, mtime: m, id: traeMemoryId(file), tool: "trae-session-memory", status: "skipped", reason: "nointent" }); continue; } + jobs.push({ file, m, project, raw, s }); } } - await pool(jobs, CONC, async ({ file, m, project, raw }) => { + await pool(jobs, CONC, async ({ file, m, project, raw, s }) => { const id = traeMemoryId(file); try { const cwd = project.cwd; const c = cwd ? coordOf(cwd, cfg.upload_folders) : { remote: null, folder: `trae/${project.slug}` }; const branch = cwd ? gitBranch(cwd) : "no-branch"; + const folder = c.folder || `trae/${project.slug}`; const slimmed = slimRaw(raw); // 复用客户端上传前脱敏/瘦身，避免 Trae memory raw 把密钥带上服务端。 - if (slimmed.length > MAX_UPLOAD) { log.warn("trae-memory 跳过：上传体超上限", { id, mb: +(slimmed.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; return; } + if (slimmed.length > MAX_UPLOAD) { log.warn("trae-memory 跳过：上传体超上限", { id, mb: +(slimmed.length / 1048576).toFixed(0) }); seenSession.set(file, m); skip++; recordSession({ file, mtime: m, id, tool: "trae-session-memory", status: "skipped", reason: "toobig", ...coordFields({ ...s, cwd }, { remote: c.remote, folder }, branch), bytes_raw: raw.length, bytes_slim: slimmed.length }); return; } const r = await post("/ingest", { - id, tool: "trae-session-memory", raw: slimmed, - remote: c.remote, folder: c.folder || `trae/${project.slug}`, branch, + id, tool: "trae-session-memory", raw: applyUserRedact(slimmed), + remote: c.remote, folder, branch, producer: cfg.me, }); - if (r.status === 200) { up++; seenSession.set(file, m); } else log.warn("trae-memory 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); + if (r.status === 200) { + up++; seenSession.set(file, m); + const sk = serverCoord(r); + recordSession({ file, mtime: m, id, tool: "trae-session-memory", status: "uploaded", ...coordFields({ ...s, cwd }, { remote: c.remote, folder }, branch), space_key: sk.space_key, server_file: sk.file, bytes_raw: raw.length, bytes_slim: slimmed.length }); + } else log.warn("trae-memory 上传失败", { id, status: r.status, body: (r.body || "").slice(0, 200) }); } catch (e) { log.error("trae-memory 出错", { id, err: e.message }); } }); return { up, skip }; } +// 历史回填：把【上线本功能前就已 seen】但账本没有的源文件，补一条派生记录，让 viewer 也能显示历史。 +// 状态按 daemon 真实判定法则重放（seen + 闸门内 + 有 intent + 不超限 → uploaded；否则 skipped+原因），故准确。 +// 大文件不细解，按大小/位置粗判，避免回填卡顿。一次性：补过的进账本，下次启动 getByFile 命中自然跳过。 +async function backfillLedger() { + let done = 0; + for (const [file, m] of seenSession) { + if (getByFile(file)) continue; // 账本已有 → 跳过 + let st; try { st = statSync(file); } catch { continue; } // 源文件没了 → 无从判定 + const tool = file.startsWith(CODEX_ROOT) ? "codex" + : file.startsWith(TRAE_MEMORY_ROOT) ? "trae-session-memory" + : file.endsWith(".md") ? "session-history-md" : "claude-code"; + const id = tool === "session-history-md" ? sessionHistoryId(file) + : tool === "trae-session-memory" ? traeMemoryId(file) + : basename(file).replace(/\.jsonl$/, ""); + const maxRaw = tool === "codex" ? MAX_RAW_CODEX : MAX_RAW; + if (st.size > maxRaw) { recordSession({ file, mtime: m, id, tool, status: "skipped", reason: "toobig", bytes_raw: st.size, backfilled: true }); done++; continue; } + if (st.size > BACKFILL_PARSE_CAP) { recordSession({ file, mtime: m, id, tool, status: "uploaded", bytes_raw: st.size, backfilled: true }); done++; continue; } // 大文件不细解，假定已传 + try { + let s; + if (tool === "codex") s = parseSessionText(await slimRawFile(file), "codex"); + else if (tool === "claude-code") s = parseSession(file); + else s = parseSessionText(readFileSync(file, "utf8"), tool); + const cwd = s.cwd || (tool === "session-history-md" ? sessionHistoryProjectDir(file) : null); + const c = coordOf(cwd, cfg.upload_folders); + const branch = s.branch || gitBranch(cwd); + let status = "uploaded", reason; // 重放判定法则（与各 sync 函数一致） + if (!s.intent) { status = "skipped"; reason = "nointent"; } + else if (s.subagent) { status = "skipped"; reason = "subagent"; } + else if (!gated(cwd)) { status = "skipped"; reason = "gated"; } + recordSession({ file, mtime: m, id, tool, status, reason, ...coordFields({ ...s, cwd }, c, branch), bytes_raw: st.size, backfilled: true }); + done++; + } catch { recordSession({ file, mtime: m, id, tool, status: "uploaded", bytes_raw: st.size, backfilled: true }); done++; } + } + if (done) { saveLedger(); log.info("[历史回填] 已补账本", { added: done }); } +} + async function tick() { const s = await syncSessions(); const x = await syncCodexSessions(); const h = await syncSessionHistoryMd(); const t = await syncTraeSessionMemory(); saveState(); // 落盘已上传记录（重启复原） + saveLedger(); // 落盘结果账本（viewer 数据源） log.info("tick", { cc_up: s.up, cc_skip: s.skip, codex_up: x.up, codex_skip: x.skip, session_history_up: h.up, session_history_skip: h.skip, trae_memory_up: t.up, trae_memory_skip: t.skip }); } @@ -481,8 +563,14 @@ function acquireLock() { const once = process.argv.includes("--once"); if (!once) acquireLock(); // 持久模式先抢锁，第二个常驻会在此退出 loadState(); // 先复原已上传记录，首轮就跳过历史 +loadLedger(LEDGER); // 复原结果账本（viewer 数据源） +loadOptout(OPTOUT); // 复原逐条排除名单 +loadUserRedact(USERREDACT); // 复原个人脱敏词表 reconcileScope(); // 范围变了就清 seen、补全新范围历史 reconcilePipeline(); // 流水线升级了就重收受影响历史（当前：Codex token 用量） log.info("sync 启动", { server: cfg.server_url, folders: (cfg.upload_folders || []).length, collect_all: COLLECT_ALL, conc: CONC, seen: seenSession.size, ver: CLIENT_VERSION, once }); +// 本机查看器（仅常驻模式起；--once 回填不起，免得拉起一个短命监听）。失败不致命：采集照常。 +if (!once) { try { startViewer({ ROOT, cfg, paths: { CC_ROOT, CODEX_ROOT, TRAE_MEMORY_ROOT } }); } catch (e) { log.warn("viewer 启动失败（不影响采集）", { err: e.message }); } } await safeTick(); +if (!once) backfillLedger().catch((e) => log.warn("历史回填异常", { err: e.message })); // 首轮上传后再回填历史，不拖慢启动 if (!once) setInterval(safeTick, (cfg.interval_sec ?? 60) * 1000); diff --git a/client/viewer.mjs b/client/viewer.mjs new file mode 100644 index 0000000..7459e03 --- /dev/null +++ b/client/viewer.mjs @@ -0,0 +1,303 @@ +// 本机查看器（M1）：采集常驻内嵌的 127.0.0.1 只读小服务，给「我的足迹」localhost 控制台供数据。 +// 只绑 loopback + 本地 token；不碰上传逻辑。数据源：结果账本（core/ledger.mjs）+ 本机源文件实时投影。 +// 预览用与服务端【同一套】core 函数（slim → projectSession），所以「脱敏后」与库里一字不差。 +import http from "node:http"; +import { readFileSync, existsSync, statSync, writeFileSync, openSync, readSync, closeSync, readdirSync } from "node:fs"; +import { join, extname, basename } from "node:path"; +import { randomBytes } from "node:crypto"; +import { parseSession } from "../core/parse.mjs"; +import { slimRaw, slimRawFile } from "../core/slim.mjs"; +import { projectSession } from "../core/project.mjs"; +import { coordOf, expandHome } from "../core/coord.mjs"; +import { allSessions, getByFile, recordSession, saveLedger } from "../core/ledger.mjs"; +import { listOptout, addOptout, removeOptout } from "../core/optout.mjs"; +import { listTerms, addTerm, removeTerm, countMatches } from "../core/userredact.mjs"; +import { log } from "../core/log.mjs"; +import { CLIENT_VERSION } from "../core/version.mjs"; +import { stringify } from "yaml"; + +const MIME = { ".html": "text/html; charset=utf-8", ".js": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".svg": "image/svg+xml", ".woff2": "font/woff2", ".ico": "image/x-icon" }; +const RAW_CAP = 400 * 1024; // 「本机原文」展示上限：截前 400KB，免得把几百 MB 的 codex 原文塞给浏览器 +const FLAG_SCAN_CAP = 40; // overview 自检：一次最多现扫 40 条未缓存的（其余等点开详情时再扫并缓存） + +export function startViewer({ ROOT, cfg, paths }) { + const WEB = join(ROOT, "web"); + const INFO = join(ROOT, ".brain-viewer.json"); + const token = ensureToken(INFO); + const ctx = { ROOT, WEB, cfg, paths, token }; + const server = http.createServer((req, res) => { + handle(req, res, ctx).catch((e) => json(res, 500, { error: String(e?.message || e) })); + }); + server.on("error", (e) => log.warn("viewer 监听异常", { err: e.message })); + listen(server, Number(cfg.viewer_port) || 7878, 0, (port) => { + writeFileSync(INFO, JSON.stringify({ port, token, url: `http://127.0.0.1:${port}/?t=${token}`, pid: process.pid })); + log.info("本机查看器已起", { url: `http://127.0.0.1:${port}/` }); + }); + return server; +} + +// 端口被占就 +1 重试（最多 20 次），只绑 127.0.0.1。 +function listen(server, port, tries, onUp) { + server.once("error", (e) => { + if (e.code === "EADDRINUSE" && tries < 20) { listen(server, port + 1, tries + 1, onUp); } + else log.warn("viewer 起不来", { err: e.message }); + }); + server.listen(port, "127.0.0.1", () => onUp(port)); +} + +function ensureToken(infoPath) { + try { const j = JSON.parse(readFileSync(infoPath, "utf8")); if (j.token) return j.token; } catch {} + return randomBytes(18).toString("base64url"); +} + +const json = (res, code, obj) => { res.writeHead(code, { "content-type": "application/json; charset=utf-8" }); res.end(JSON.stringify(obj)); }; + +async function handle(req, res, ctx) { + const u = new URL(req.url, "http://127.0.0.1"); + const p = u.pathname; + + // 静态：页面 + 资源（loopback only，页面本身不校验 token；token 在 /api 上把关） + if (p === "/" ) return serveFile(res, join(ctx.WEB, "viewer.html")); + if (!p.startsWith("/api/")) { + const f = join(ctx.WEB, p.replace(/^\/+/, "")); + if (f.startsWith(ctx.WEB) && existsSync(f) && statSync(f).isFile()) return serveFile(res, f); + return json(res, 404, { error: "not found" }); + } + + // API：本地 token 校验（query t 或 header） + const tok = u.searchParams.get("t") || req.headers["x-brain-token"]; + if (tok !== ctx.token) return json(res, 401, { error: "bad local token" }); + + // GET + if (req.method === "GET") { + if (p === "/api/overview") return json(res, 200, overview(ctx)); + if (p === "/api/sessions") return json(res, 200, sessions(ctx)); + if (p === "/api/session") return json(res, 200, await sessionDetail(u.searchParams.get("file"))); + if (p === "/api/log") return json(res, 200, { lines: tailLog(ctx.ROOT, 1000).filter(meaningfulLog).slice(-200) }); + if (p === "/api/optout") return json(res, 200, { entries: listOptout() }); + if (p === "/api/redact") return json(res, 200, redactInfo()); + return json(res, 404, { error: "not found" }); + } + // POST（前端发的是普通 JSON，不 gzip） + if (req.method === "POST") { + let body = {}; + try { body = JSON.parse((await readReq(req)) || "{}"); } catch { return json(res, 400, { error: "bad json" }); } + if (p === "/api/exclude") return json(res, 200, await excludeSession(ctx, body.file)); + if (p === "/api/unexclude") return json(res, 200, (removeOptout({ id: body.id, file: body.file }), { ok: true })); + if (p === "/api/redact-add") return json(res, 200, (addTerm({ pattern: body.pattern, type: body.type }), { ok: true })); + if (p === "/api/redact-remove") return json(res, 200, (removeTerm(body.pattern), { ok: true })); + if (p === "/api/config-dryrun") return json(res, 200, configDryrun(ctx, body)); + if (p === "/api/config") return json(res, 200, saveConfig(ctx, body)); + return json(res, 404, { error: "not found" }); + } + return json(res, 404, { error: "not found" }); +} + +const readReq = (req) => new Promise((resolve, reject) => { let d = ""; req.on("data", (c) => (d += c)); req.on("end", () => resolve(d)); req.on("error", reject); }); + +function serveFile(res, f) { + try { + res.writeHead(200, { "content-type": MIME[extname(f)] || "application/octet-stream" }); + res.end(readFileSync(f)); + } catch { json(res, 404, { error: "not found" }); } +} + +// ---------- 闸门（与 sync.mjs 同逻辑，pending 实时扫描用）---------- +const within = (path, f) => { const r = expandHome(f); return !!r && (path === r || path.startsWith(r.endsWith("/") ? r : r + "/")); }; +function gateOf(cfg) { + const excluded = (path) => (cfg.exclude || []).some((e) => within(path, e)); + const collectAll = !!cfg.collect_all; + return (cwd) => !!cwd && !excluded(cwd) && (collectAll || (cfg.upload_folders || []).some((f) => within(cwd, f))); +} + +// ---------- 坐标显示串 ---------- +function coordStr(e) { + if (e.space_key) return `${e.space_key}${e.branch ? " · " + e.branch : ""}`; + const r = e.remote; + const base = r ? `${r.host}/${r.owner}/${r.repo}` : (e.folder ? `local · ${e.folder}` : "local"); + return `${base}${e.branch ? " · " + e.branch : ""}`; +} +const toRow = (e) => ({ + file: e.file, id: e.id, tool: toolShort(e.tool), intent: e.intent || "（无意图）", + coord: coordStr(e), time: e.work_end || e.work_start || null, status: e.status, +}); +const toolShort = (t) => ({ "claude-code": "CC", "codex": "Codex", "trae-session-memory": "Trae", "session-history-md": "MD" }[t] || t); + +// ---------- overview ---------- +function overview(ctx) { + const all = allSessions(); + const counts = { uploaded: 0, skipped: 0 }; + for (const e of all) if (e.status === "uploaded") counts.uploaded++; else if (e.status === "skipped") counts.skipped++; + const pending = scanPending(ctx); + return { + me: ctx.cfg.me || {}, version: CLIENT_VERSION, server_url: ctx.cfg.server_url || "", + lastSync: lastTick(ctx.ROOT), + counts: { uploaded: counts.uploaded, pending: pending.length, skipped: counts.skipped, localTotal: all.length + pending.length }, + config: pickConfig(ctx.cfg), + }; +} +function pickConfig(cfg) { + return { + collect_all: !!cfg.collect_all, upload_folders: cfg.upload_folders || [], exclude: cfg.exclude || [], + codex: cfg.codex !== false, trae_memory: cfg.trae_memory !== false, session_history_md: cfg.session_history_md !== false, + interval_sec: cfg.interval_sec ?? 60, debounce_sec: cfg.debounce_sec ?? 60, + }; +} + +// ---------- sessions（已在库 + 实时派生的待传）---------- +function sessions(ctx) { + const uploaded = allSessions().filter((e) => e.status === "uploaded") + .sort((a, b) => String(b.work_end || "").localeCompare(String(a.work_end || ""))).map(toRow); + return { uploaded, pending: scanPending(ctx) }; +} + +// 「即将上传」实时派生：扫 CC/Codex 在去抖窗口内（还没稳定、下一轮才会传）且闸门内、尚未上传的文件。 +// 不落账本（瞬态），只在 viewer 现算；按 mtime 近期性先筛、再 parse，控制开销。 +function scanPending(ctx) { + const gated = gateOf(ctx.cfg); + const debounceMs = (ctx.cfg.debounce_sec ?? 60) * 1000; + const out = []; + const consider = (file, tool) => { + let st; try { st = statSync(file); } catch { return; } + if (Date.now() - st.mtimeMs >= debounceMs) return; // 已稳定 → 不算待传（要么已传、要么已被处理） + const e = getByFile(file); if (e && e.status === "uploaded" && e.mtime === st.mtimeMs) return; + if (st.size > 8 * 1024 * 1024) return; // 太大不在 pending 里现 parse + let s; try { s = parseSession(file); } catch { return; } + if (!s.intent || !gated(s.cwd)) return; + const c = coordOf(s.cwd, ctx.cfg.upload_folders); + out.push({ file, id: basename(file).replace(/\.jsonl$/, ""), tool: toolShort(tool), intent: s.intent, + coord: coordStr({ remote: c.remote, folder: c.folder, branch: s.branch }), time: s.updated || s.ts, status: "pending", flags: null }); + }; + const { CC_ROOT, CODEX_ROOT } = ctx.paths || {}; + if (CC_ROOT && existsSync(CC_ROOT)) for (const proj of safeDir(CC_ROOT)) for (const f of safeDir(join(CC_ROOT, proj))) if (f.endsWith(".jsonl")) consider(join(CC_ROOT, proj, f), "claude-code"); + if (CODEX_ROOT && ctx.cfg.codex !== false) for (const f of walkCodex(CODEX_ROOT)) consider(f, "codex"); + return out; +} +const safeDir = (d) => { try { return readdirSync(d); } catch { return []; } }; +function* walkCodex(dir) { + for (const e of safeDir(dir)) { + const p = join(dir, e); + let st; try { st = statSync(p); } catch { continue; } + if (st.isDirectory()) yield* walkCodex(p); + else if (e.startsWith("rollout-") && e.endsWith(".jsonl")) yield p; + } +} + +// ---------- 单条详情：本地投影出「脱敏后」正文（与库里一致）+ 原文 + 自检 ---------- +async function sessionDetail(file) { + if (!file) return { error: "missing file" }; + const e = getByFile(file); + const tool = e?.tool || "claude-code"; + if (!existsSync(file)) return { error: "源文件已不在本机（可能已删除/移动）", id: e?.id, coord: e ? coordStr(e) : "" }; + let uploadStr; + try { + if (tool === "codex") uploadStr = await slimRawFile(file); + else if (tool === "session-history-md") uploadStr = readFileSync(file, "utf8"); // session-history 上传不蒸馏 + else uploadStr = slimRaw(readFileSync(file, "utf8")); + } catch (err) { return { error: "读取/蒸馏失败：" + err.message }; } + const redacted = projectSession(uploadStr, tool); + const { text: raw, truncated } = readCapped(file); + return { + id: e?.id || basename(file).replace(/\.(jsonl|md)$/, ""), tool: toolShort(tool), + intent: e?.intent || "", coord: e ? coordStr(e) : "", status: e?.status || "uploaded", + redacted, raw, truncated, + }; +} + +// 读文件前 RAW_CAP 字节（大文件不整读） +function readCapped(file) { + try { + const size = statSync(file).size; + if (size <= RAW_CAP) return { text: readFileSync(file, "utf8"), truncated: false }; + const fd = openSync(file, "r"); const buf = Buffer.alloc(RAW_CAP); + const n = readSync(fd, buf, 0, RAW_CAP, 0); closeSync(fd); + return { text: buf.subarray(0, n).toString("utf8") + "\n\n…（原文超 400KB，仅展示开头）", truncated: true }; + } catch (e) { return { text: "（读取失败：" + e.message + "）", truncated: false }; } +} + +// ---------- M2：排除 / 撤回 ---------- +// 排除一条：加进本机 optout（daemon 上传前据此跳过）；若已在库，再调服务端 /retract 删掉。 +async function excludeSession(ctx, file) { + if (!file) return { error: "missing file" }; + const e = getByFile(file); + const id = e?.id || basename(file).replace(/\.(jsonl|md)$/, ""); + addOptout({ id, file, intent: e?.intent || null }); + let retract = null; + if (e && e.status === "uploaded") { + retract = await callRetract(ctx.cfg, id); + recordSession({ file, status: retract?.ok ? "retracted" : "opted_out" }); + } else if (e) { + recordSession({ file, status: "opted_out" }); + } + saveLedger(); + return { ok: true, retract }; +} +async function callRetract(cfg, id) { + if (!cfg.server_url || !cfg.token) return { ok: false, error: "未配置服务器 / token" }; + try { + const r = await fetch(cfg.server_url.replace(/\/$/, "") + "/retract", { + method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${cfg.token}` }, body: JSON.stringify({ id }), + }); + const j = await r.json().catch(() => ({})); + return r.ok ? { ok: true, removed: (j.removed || []).length } : { ok: false, error: j.error || ("HTTP " + r.status) }; + } catch (e) { return { ok: false, error: e.message }; } +} + +// ---------- M2：个人脱敏词表（含本机命中计数）---------- +function redactInfo() { + const builtin = ["API Key (sk-…)", "GitHub PAT", "GitLab Token", "AWS", "Google", "Slack", "Stripe", "JWT", "私钥 (PEM)", "password/token 赋值", "URL 内账密", "家目录路径 → ~"]; + const terms = listTerms().map((t) => ({ ...t, count: 0, sessions: 0 })); + if (terms.length) { + let scanned = 0; + for (const e of allSessions()) { + if (e.status !== "uploaded" || e.tool === "codex" || scanned >= FLAG_SCAN_CAP || !existsSync(e.file)) continue; + let text; try { text = e.tool === "session-history-md" ? readFileSync(e.file, "utf8") : slimRaw(readFileSync(e.file, "utf8")); } catch { continue; } + scanned++; + for (const t of terms) { const n = countMatches(text, t); if (n) { t.count += n; t.sessions++; } } + } + } + return { builtin, terms }; +} + +// ---------- M2：配置预览差量 / 保存 ---------- +function configDryrun(ctx, proposed) { + const cur = gateOf(ctx.cfg); + const prop = gateOf({ collect_all: !!proposed.collect_all, upload_folders: proposed.upload_folders || [], exclude: proposed.exclude || [] }); + let willAdd = 0, willStop = 0; + for (const e of allSessions()) { + if (!e.cwd) continue; + const a = cur(e.cwd), b = prop(e.cwd); + if (!a && b) willAdd++; + else if (a && !b && e.status === "uploaded") willStop++; + } + return { willAdd, willStop, basis: "按本机账本里已知 cwd 的 session 估算" }; +} +function saveConfig(ctx, body) { + const cfg = ctx.cfg; + for (const k of ["collect_all", "upload_folders", "exclude", "codex", "trae_memory", "session_history_md", "interval_sec", "debounce_sec"]) { + if (k in body) cfg[k] = body[k]; + } + try { + writeFileSync(join(ctx.ROOT, "client.config.yaml"), stringify(cfg)); + return { ok: true, note: "已保存。开关类即时生效；采集范围(目录)改动在 brain service restart 后完整补扫历史。" }; + } catch (e) { return { ok: false, error: e.message }; } +} + +// ---------- 日志 ---------- +// 过滤「无事发生」的空转 tick：tick 行里所有计数都是 0 → 丢；非 tick 行（上传/跳过/错误/启动/回填等）保留。 +function meaningfulLog(line) { + if (!/\btick\b/.test(line)) return true; + const nums = [...line.matchAll(/=(\d+)/g)].map((m) => Number(m[1])); + return nums.length === 0 || nums.some((n) => n !== 0); +} +function tailLog(ROOT, n) { + try { + const lines = readFileSync(join(ROOT, "sync.log"), "utf8").trimEnd().split("\n"); + return lines.slice(-n); + } catch { return []; } +} +function lastTick(ROOT) { + const lines = tailLog(ROOT, 400); + for (let i = lines.length - 1; i >= 0; i--) if (lines[i].includes(" tick")) return lines[i]; + return lines[lines.length - 1] || null; +} diff --git a/core/ledger.mjs b/core/ledger.mjs new file mode 100644 index 0000000..f866c8c --- /dev/null +++ b/core/ledger.mjs @@ -0,0 +1,40 @@ +// 结果账本（M1 地基）：记录采集常驻对【每条 session 源文件】最终做了什么 —— 上传 / 跳过(原因) / 排除。 +// 与 .brain-state.json（只记 mtime 做去重）分开：账本要回答「传了啥 / 没传啥 / 为什么 / 在库哪」， +// 是本机 localhost 查看器（client/viewer.mjs）的唯一数据源。纯本机文件，不上传。 +// +// 一条记录 = 一个源文件（key=file 绝对路径）： +// { file, id, tool, status, reason?, intent?, cwd?, remote?, folder?, branch?, +// space_key?, server_file?, bytes_raw?, bytes_slim?, work_start?, work_end?, +// mtime?, recorded_at } +// status ∈ uploaded | skipped | opted_out （pending 由 viewer 实时扫描派生，不落账本） +// reason ∈ gated | excluded | toobig | nointent | subagent | empty | error （status=skipped 时） +import { readFileSync, writeFileSync } from "node:fs"; + +let FILE = null; +const map = new Map(); // file -> record + +export function loadLedger(path) { + FILE = path; + try { + const j = JSON.parse(readFileSync(path, "utf8")); + for (const e of j.sessions || []) if (e && e.file) map.set(e.file, e); + } catch { /* 首次 / 损坏 → 空账本，照常往下走 */ } +} + +export function saveLedger() { + if (!FILE) return; + try { writeFileSync(FILE, JSON.stringify({ v: 1, sessions: [...map.values()] })); } + catch { /* 落盘失败不致命：下一轮再写 */ } +} + +// 记一条（按 file 合并，保留已知字段、覆盖新值）。rec.file 必填。 +export function recordSession(rec) { + if (!rec || !rec.file) return; + const prev = map.get(rec.file) || {}; + map.set(rec.file, { ...prev, ...rec, recorded_at: Date.now() }); +} + +export function allSessions() { return [...map.values()]; } +export function getByFile(file) { return map.get(file) || null; } +export function getById(id) { for (const e of map.values()) if (e.id === id) return e; return null; } +export function ledgerSize() { return map.size; } diff --git a/core/optout.mjs b/core/optout.mjs new file mode 100644 index 0000000..8fc2417 --- /dev/null +++ b/core/optout.mjs @@ -0,0 +1,30 @@ +// 逐条排除（opt-out）：本机名单，采集常驻上传前据此跳过；viewer 增删。纯本机文件，不上传。 +// 按 session id 或源文件绝对路径匹配（id 跨机器稳定；file 是本机路径）。 +// 与 viewer 同进程 → viewer 改了名单，daemon 下一轮即生效，无需重启。 +import { readFileSync, writeFileSync } from "node:fs"; + +let FILE = null; +let entries = []; // [{ id?, file?, intent?, at }] +const byId = new Set(), byFile = new Set(); + +function reindex() { byId.clear(); byFile.clear(); for (const e of entries) { if (e.id) byId.add(e.id); if (e.file) byFile.add(e.file); } } + +export function loadOptout(path) { + FILE = path; + try { entries = (JSON.parse(readFileSync(path, "utf8")).entries) || []; } catch { entries = []; } + reindex(); +} +function save() { if (!FILE) return; try { writeFileSync(FILE, JSON.stringify({ v: 1, entries })); } catch {} } + +export function isOptedOut(id, file) { return !!((id && byId.has(id)) || (file && byFile.has(file))); } +export function addOptout(e) { + if (!e || (!e.id && !e.file)) return; + if (isOptedOut(e.id, e.file)) return; + entries.push({ id: e.id || null, file: e.file || null, intent: e.intent || null, at: Date.now() }); + reindex(); save(); +} +export function removeOptout({ id, file }) { + entries = entries.filter((e) => !((id && e.id === id) || (file && e.file === file))); + reindex(); save(); +} +export function listOptout() { return entries.slice(); } diff --git a/core/userredact.mjs b/core/userredact.mjs new file mode 100644 index 0000000..6f72568 --- /dev/null +++ b/core/userredact.mjs @@ -0,0 +1,43 @@ +// 个人脱敏词表：在内置自动脱敏（core/redact.mjs）【之后】，再把用户自定义的词 / 正则替换为 [REDACTED]。 +// 补内置规则覆盖不到的「内容级」敏感信息（客户名、真名、内部代号）。纯本机、不上传；上传前应用，只影响新上传。 +import { readFileSync, writeFileSync } from "node:fs"; + +let FILE = null; +let terms = []; // [{ pattern, type:'text'|'regex' }] +let compiled = []; // [RegExp] + +const escapeRe = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +// '/foo/i' → RegExp；普通文本 → 全局字面量。统一带 g 以便 replace-all 与计数。 +function toRe(t) { + if (t.type === "regex") { + const m = String(t.pattern).match(/^\/(.*)\/([gimsuy]*)$/); + if (m) return new RegExp(m[1], m[2].includes("g") ? m[2] : m[2] + "g"); + return new RegExp(t.pattern, "g"); + } + return new RegExp(escapeRe(t.pattern), "g"); +} +function compile() { compiled = []; for (const t of terms) { try { compiled.push(toRe(t)); } catch {} } } + +export function loadUserRedact(path) { + FILE = path; + try { terms = (JSON.parse(readFileSync(path, "utf8")).terms) || []; } catch { terms = []; } + compile(); +} +function save() { if (!FILE) return; try { writeFileSync(FILE, JSON.stringify({ v: 1, terms })); } catch {} } + +export function applyUserRedact(text) { + if (!compiled.length) return text; // 没词条 → 原样返回（零开销） + let s = String(text); + for (const re of compiled) s = s.replace(re, "[REDACTED]"); + return s; +} +export function hasUserRedact() { return compiled.length > 0; } +export function listTerms() { return terms.slice(); } +export function addTerm(t) { + if (!t || !t.pattern || terms.some((x) => x.pattern === t.pattern)) return; + terms.push({ pattern: t.pattern, type: t.type === "regex" ? "regex" : "text" }); + save(); compile(); +} +export function removeTerm(pattern) { terms = terms.filter((t) => t.pattern !== pattern); save(); compile(); } +// 某词条在给定文本里命中多少次（viewer 显示「本机命中 N 条」用）。 +export function countMatches(text, t) { try { const m = String(text).match(toRe(t)); return m ? m.length : 0; } catch { return 0; } } diff --git a/docs/PRODUCER_FOOTPRINT.md b/docs/PRODUCER_FOOTPRINT.md new file mode 100644 index 0000000..6ae4e88 --- /dev/null +++ b/docs/PRODUCER_FOOTPRINT.md @@ -0,0 +1,161 @@ +# 生产者足迹 / 上传前预览 / 按条 opt-out 设计 + +> 目标：降低被动入库者的「被监视感」，让全员愿意开。 +> 分支：`feat-producer-footprint` + +## 1. 为什么做这个 + +team-brain 的价值靠**全员开**才成立——一个人不开，他参与的跨人项目就有黑洞。 +但「我干活的全文对话被持续上传、队友随时能 grep」天然让人不安。现状的安全感 +全压在**安装时**那一道闸（`upload_folders` 白名单 / `exclude` 黑名单 / `collect_all` 警告）， +开了之后就是个**黑盒**： + +- 我**不知道**我哪些 session 真进了库（`brain status` 只给最近一条 tick 的计数）。 +- 我**看不到**脱敏后长啥样——只能信「客户端会脱敏」，但 `redact.mjs` 自己注明 + *内容级脱敏（客户名等）尚未做*。 +- 我想撤一条**做不到**：opt-out 只有文件夹级（`exclude`），没有「就这一条别传」。 + +把黑盒变成**可见 + 可控**，是这版要解决的。 + +## 2. 关键现状（决定哪些便宜、哪些贵） + +调研后三条结论直接决定方案形状（证据见 `client/sync.mjs`、`core/`、`server/`）： + +| 结论 | 证据 | 对设计的影响 | +|---|---|---| +| **预览几乎免费** | `core/redact.mjs` / `core/project.mjs`(`projectSession=redactAgent(transcript)`) / `core/slim.mjs` 全是**确定性、无副作用**函数，且 `core/` **本就随客户端下发**（`client.tgz`=core/client/mcp/cli）。 | 客户端能本地跑 `projectSession(slimRaw(raw), tool)` 复现**与服务端一字不差**的 `.md` 正文。**零服务端改动**。 | +| **足迹主要是接线** | 服务端已有 `/whoami`（token→自己 id）+ `/sessions?author=`（按人列 session）。 | 「库里到底有我哪些」是一次查询。缺口仅在客户端：`.brain-state.json` 把**上传**和**跳过**标成一样（到处 `seenSession.set(file,m)`），丢掉了「传没传/为啥」。 | +| **按条 opt-out 是新东西，且便宜的一半在上传前** | 无任何 DELETE/retract 端点；`admin remove` 只撤 token、**保留全部历史**。脱敏/闸门均在上传前（`sync.mjs:166/204/389`）。 | 「这条永不上传」= 纯客户端新增。「撤一条已传的」= 新服务端端点 + git 语义，明显更贵、且涉及 git 历史。 | + +## 3. 设计原则 + +1. **信任的最强杠杆是「上传前的本地控制」**——东西没离开机器，焦虑最小。 + 优先做 client 侧、CLI（`brain` 是本地、已有的唯一触点）。Web 是协作/共享面、 + 且**没有 per-viewer 身份**（任何 token 能读任何人），放后面。 +2. **先可见，再可控**：透明本身就降焦虑且零风险（不改上传行为）。 +3. **不改默认契约**：保持「默认上传 + opt-out」的现有气质，opt-out 是**加法**； + 是否引入「先审后传」作为 opt-in 单独决策（见 §7）。 +4. **真相库幂等、git 可追溯**：retract 要诚实标注「git 历史里仍可考古，除非额外重写历史」。 + +## 4. 三件事的具体方案 + +### 4.0 地基：把「结果」记下来（`outcome ledger`） +现状 daemon 处理完每个文件就 `seenSession.set(file, m)`，无论是**上传成功**、 +**闸门外跳过**、**太大跳过**、**无 intent 跳过**——信息全丢。 + +改动：在 `.brain-state.json` 旁加一份**结果账本**（或把 `session` 值从 `mtime` +升级成 `{mtime, status, ...}`，向后兼容旧格式）。每条记： + +``` +{ file, id, tool, status, space_key?, branch?, intent, bytes_raw, bytes_slim, ts } +status ∈ uploaded | skipped_gated | skipped_excluded | skipped_toobig + | skipped_nointent | skipped_subagent | opted_out +``` + +这是**一切的地基**且纯客户端：`brain mine` 靠它离线回答「传了啥 / 没传啥 / 为啥」。 +落点：`client/sync.mjs`（4 个 sync* 函数的标记点）+ 一个小 `core/ledger.mjs`。 + +### 4.1 我的足迹 —— `brain mine`（别名 `brain footprint`） +**本地视角（默认，离线）**：读结果账本，分组列出 +``` +$ brain mine +本机最近 50 条 session： + ✓ 已传 12 ⤫ 闸门外 30 ⛔ 已排除 3 🚫 我手动排除 2 ⚠ 跳过(大/空) 1 + +✓ github__Asklear__bossa / main + tqt-...901a "理解 staff bot 任务管理…" 2026-06-24 →库 + ... +⤫ 闸门外（留本机，没传） + ~/Code/private-x "改个支付密钥…" （不在 upload_folders） +``` +每行可接动作：`brain preview ` / `brain optout `。 + +**库内视角（`--remote`，对账）**：`/whoami` → `/sessions?author=me`，列**库里实际有的**。 +用途：抓「别的机器传的」「以为撤了其实还在」。本地账本 ⨯ 库内清单的差集 = 惊喜来源。 + +### 4.2 上传前预览 —— `brain preview ` +因为脱敏+渲染都在 `core/` 且确定性，客户端**本地**算出与库里一字不差的正文： +```js +import { slimRaw } from "../core/slim.mjs"; +import { projectSession } from "../core/project.mjs"; +const slim = slimRaw(readFileSync(file, "utf8")); // = 上传体 +const body = projectSession(slim, tool); // = 服务端 .md 正文（脱敏后） +``` +输出： +- **队友会看到的正文**（脱敏后 transcript）， +- 会落到哪个坐标（space/branch，由 `coordOf` + 现状闸门判定）， +- **被剥/被抹了什么**的提示（图片/大输出截断、命中的脱敏规则计数）， +- **脱敏自检告警**：扫一遍残留的邮箱/疑似人名/疑似客户名等—— + 对应 `redact.mjs` 自承「内容级脱敏未做」，至少给红旗，别假装干净。 + +可选 `brain preview --raw` 看蒸馏后的 `.jsonl` 上传体本身。 + +### 4.3 按条 opt-out —— `brain optout` +**上传前（便宜、纯客户端、信任主力）**： +- 本地 opt-out 名单：`~/.team-brain/optout`（gitignore 范畴的本机文件）， + 记 session id / 文件路径 / glob。daemon 在 `gated()` 之外再过一道 + `optedOut(file|id)`，命中即**永不上传**并在账本记 `opted_out`。 +- CLI：`brain optout ` 加；`brain optout --list`；`brain optout --rm `。 + `brain mine` 行内也能直接 opt-out。 +- 顺带支持一个 `.team-brain-ignore`（gitignore 风格、按 session 文件 glob）， + 补 `exclude`（子树级）够不到的**文件级**粒度。 + +**已传后撤回 —— `brain retract `（贵、需服务端、Phase 3）**： +- 新端点 `POST /retract`（token 鉴权，**只能撤自己 producer_id 的**）： + git 删除该 session 的 `.md`/`.jsonl` 并 commit。 +- 诚实告知：默认只从 HEAD 移除，**git 历史仍可考古**；要真抹需额外历史重写（重活、慎做）。 +- 同时把该 id 自动加入本机 opt-out，防 daemon 下一轮又传回去。 + +## 5. 顺手降低「默认就吓人」 +`collect_all`（安装没填工作空间时的默认）最吓人。首次 `--once` 回填后给一条总结： +> 这台机器将上传 N 个项目、约 M 条 session。`brain mine` 看明细，`brain optout` 排除。 + +一次性 onboarding nudge，把「不知不觉全传了」变成「我知道、我能改」。 + +## 5b. 已拍板的方向（讨论收敛） + +- **默认仍自动传**：不引入「先审后传」hold 模式。viewer 是**事后**看 + 管 + 随时排除/撤回。 +- **主界面 = 本机 localhost 查看器**（127.0.0.1，仅本人可见），把预览 / 排除 / 撤回 / 配置 / 日志揉在一处。 + 与线上共享库强视觉区分（深色 ribbon「本机控制台」）。原型见 [`web/viewer-mock.html`](../web/viewer-mock.html)。 +- **控制是事后式**：逐条「排除」（未传的永不传 / 已传的撤回）+「编辑」（overlay 二次脱敏）。 +- **脱敏两层都暴露**：内置自动规则只读展示 + 个人脱敏词表（自定义、只存本机、带本机命中计数）。 +- 中心 Web 暂不做「我的足迹」（足迹在本地看）；服务端只负责后期的**配置快照上报**（透明）。 +- localhost 安全：仅绑 loopback + 本地 token；客户端保持轻量（Node 内置 `http`，不引 express）。 + +## 6. 开发计划（里程碑） + +> 状态（截至本轮）：**M1 ✅ 完成**、**M2 ✅ 基本完成**（除 overlay 手改编辑器外全部落地），M3 待做。 +> 落地代码：`core/ledger.mjs`·`core/selfcheck.mjs`·`core/optout.mjs`·`core/userredact.mjs`·`client/viewer.mjs`·`server/retract.mjs`·`web/viewer.{html,js}`；改 `client/sync.mjs`·`cli/brain.mjs`·`server/server.mjs`。 +> 验证：全套 144 测试通过；viewer 6 个 API + 排除→服务端撤回→文件删除全链路、个人词表命中计数、配置 dry-run 均实测通过。 +> 仍缺：**overlay 手改编辑器**（逐 turn/逐段手动划掉）—— 个人脱敏词表已覆盖大部分「手动兜底」诉求，overlay 留作 M3。 + + +### M1 · 地基 + 只读足迹（纯客户端，零服务端，零行为变更） +打开黑盒。 +- `core/ledger.mjs`（新）：结果账本状态机 `uploaded | pending | skipped(reason) | opted_out` + 元信息（坐标/intent/体积/flags）。 +- 改 `client/sync.mjs`：每条处理后写账本，替换现在无差别的 `seenSession.set(file,m)`；兼容旧 `.brain-state.json`。 +- `client/viewer.mjs`（新）：Node `http` 起 127.0.0.1 + 本地 token，serve `web/` + JSON API： + `/api/overview`、`/api/sessions`、`/api/session/:id`（本地 `projectSession(slimRaw(file))` 出**真**预览 + 原文 + 脱敏自检）、`/api/log`（读 `sync.log`）。 + （嵌进常驻 daemon 还是独立进程 = M1 第一个工程决定） +- `cli/brain.mjs`：`brain viewer`（开浏览器）；`brain status` 打印 URL。 +- `web/`：把 mock 接成 fetch 真数据。 +- 脱敏自检：扫邮箱 / 疑似人名 / 长数字串 → flags。 + +### M2 · 控制（排除 + 撤回 + 配置编辑 + 个人词表） +- 逐条**排除**：本地 optout 名单；daemon 在 `gated()` 前过滤；记账本 `opted_out`。 +- **撤回**（已传）：新端点 `POST /retract`（token 鉴权，只允许撤自己 `producer_id`）→ git rm `.md`/`.jsonl` + commit；client + viewer 按钮；自动加 optout 防重传。**诚实标注 git 历史仍可考古**。 +- **配置可视化**：viewer 写 `client.config.yaml`；保存前 dry-run「多传 N / 少传 M」。 +- **个人脱敏词表**：本地存（`~/.team-brain/redact.local`）；接进上传前 slim/redact 流水（决定：仅对新上传生效，还是回溯重传已传的）。 +- overlay 编辑器（手动二次脱敏，不碰源文件）——此处或挪 M3。 + +### M3 · 透明 + 打磨 +- 服务端 **config-report**：client 心跳上报有效配置（姿态，**不含**敏感：exclude 路径/optout 内容/词表只报数量）；服务端 per-person 存；中心 web 加「全队配置 / 覆盖度」页（对等透明）。 +- overlay 编辑器（若未做）。 +- 测试（ledger / viewer API / retract / 个人词表）、文档（README + AGENTS 不变量）、灰度。 + +**建议从 M1 起跑**：纯客户端、零行为变更，立刻把黑盒打开。M2 再上控制（含唯一的服务端新增 `/retract`），M3 做团队级透明。 + +## 7. 仍需边做边定的工程点 +1. viewer 嵌入常驻 daemon vs 独立进程（单例锁 / `--once` 交互）。 +2. 个人词表对**已上传**历史是否回溯重传。 +3. 撤回语义：HEAD 移除够不够，还是要抹 git 历史。 diff --git a/server/retract.mjs b/server/retract.mjs new file mode 100644 index 0000000..8f2b811 --- /dev/null +++ b/server/retract.mjs @@ -0,0 +1,33 @@ +// 撤回：把某条 session 从真相库删掉（git rm + commit）。只允许撤【自己 producer-id】的 —— base 前缀必须等于 +// 调用者的 member.id。诚实告知：默认只从 HEAD 移除，git 历史里仍可考古（要彻底抹需另做历史重写，慎重）。 +import { join, relative } from "node:path"; +import { readdirSync, unlinkSync } from "node:fs"; +import { commit } from "./gitstore.mjs"; +import { log } from "../core/log.mjs"; + +export async function retract(truthDir, payload, member) { + const id = payload?.id; + if (!id) throw new Error("missing id"); + const base = `${member.id}-${id}`; // 只能撤自己的：前缀锁死成调用者 id + const removed = []; + const stack = [join(truthDir, "spaces")]; + while (stack.length) { + const d = stack.pop(); + let es; try { es = readdirSync(d, { withFileTypes: true }); } catch { continue; } + for (const e of es) { + if (e.name === ".git") continue; + const p = join(d, e.name); + if (e.isDirectory()) { stack.push(p); continue; } + if (e.name === `${base}.jsonl` || e.name === `${base}.md`) { + try { unlinkSync(p); removed.push(relative(truthDir, p)); } catch {} + } + } + } + if (!removed.length) return { removed: [] }; + const sha = await commit(truthDir, { + name: member.name, email: member.email || `${member.id}@team-brain`, + message: `retract ${base}`, paths: removed, + }); + log.info("retract", { who: member.id, id, files: removed.length, commit: sha }); + return { removed, commit: sha }; +} diff --git a/server/server.mjs b/server/server.mjs index b7f3266..a1e3b96 100644 --- a/server/server.mjs +++ b/server/server.mjs @@ -21,6 +21,7 @@ import { loadNotion, makeReq as makeNotionReq } from "../core/notion.mjs"; import { loadGoogle, makeReq as makeGoogleReq } from "../core/google.mjs"; import { initTruth } from "./gitstore.mjs"; import { ingest } from "./ingest.mjs"; +import { retract } from "./retract.mjs"; import { refreshAll, enumAndRegisterOrgRepos } from "./codestate.mjs"; import { readSpaceMeta } from "./space.mjs"; import { syncFeishuDocs } from "./feishudocs.mjs"; @@ -113,8 +114,11 @@ function buildClientTarball() { stage = mkdtempSync(join(tmpdir(), "tb-client-")); writeFileSync(join(stage, "package.json"), JSON.stringify(clientPkg, null, 2) + "\n"); // tar 多个 -C：代码取自 ROOT，package.json 取自 stage（GNU/BSD tar 都支持）。 + // web 只带本机查看器那两个文件（viewer.html/js，~44K，CSS 已内联、无外链字体）； + // 绝不带 web/fonts、web/app.* —— 那是线上服务端查看器，server-only，会把装机包撑肥。 execFileSync("tar", ["czf", CLIENT_TGZ, "-C", ROOT, "core", "client", "mcp", "cli", "install.sh", "client.config.example.yaml", + "web/viewer.html", "web/viewer.js", "-C", stage, "package.json"], { stdio: "ignore" }); log.info("[client] 已打包客户端", { path: CLIENT_TGZ }); } catch (e) { log.warn("[client] 打包失败", { err: e.message }); } @@ -511,6 +515,21 @@ async function handle(req, res, u) { } } + // --- retract（撤回自己的 session）--- + if (req.method === "POST" && u.pathname === "/retract") { + const member = authMember(req); + if (!member) return json(res, 401, { error: "invalid or missing token" }); + let payload; + try { payload = JSON.parse(await readBody(req)); } catch { return json(res, 400, { error: "bad json" }); } + try { + const r = await retract(TRUTH, payload, member); + return json(res, 200, { ok: true, ...r }); + } catch (e) { + log.error("retract failed", { who: member.id, id: payload?.id, err: e?.message }); + return json(res, 500, { error: "retract 失败（详情见服务器日志）" }); + } + } + json(res, 404, { error: "not found" }); } diff --git a/web/viewer.html b/web/viewer.html new file mode 100644 index 0000000..7ac1579 --- /dev/null +++ b/web/viewer.html @@ -0,0 +1,193 @@ + + + + + +Team Brain · Local Console + + + + + +
+
Local console127.0.0.1 · only you can see this · not the shared library
+
Online shared library
+
+ +
+ + +
+
+

Overview

+

What this machine has published to the team's shared library — at a glance, retractable anytime.

+
+ +
This page lives only on your machine (127.0.0.1) and only you can see it. It governs what the collector uploads from this computer to the team's shared library. Preview and exclude happen locally; only Retract reaches the shared library to delete an entry.
+
+
+

Counts are reconstructed from sessions still on this machine, so they can be fewer than the shared library (which keeps everything ever uploaded, including sessions whose local files were since removed). The shared library is the source of truth.

+
Recent activity
+
+
+ +
+

My Sessions

+

Conversations already in / about to enter the shared library. Click any to see exactly what teammates see.

+
StatusIntentToolLocationWorked
+

Only sessions in / entering the library are listed. Out-of-scope, skipped, or retracted ones are hidden.

+
+ +
+

Activity

+

What the collector has been doing recently — same source as `brain logs`. Idle ticks (nothing happened) are hidden.

+
+
+ +
+

Collection Settings

+

What's collected, where it goes, what's masked before upload. All changes affect this machine only.

+
+
+
+
+ +
+ +
+ + + + diff --git a/web/viewer.js b/web/viewer.js new file mode 100644 index 0000000..d0aad20 --- /dev/null +++ b/web/viewer.js @@ -0,0 +1,272 @@ +// 本机足迹查看器前端。默认英文，可切中文（localStorage tb_lang）。数据来自常驻内嵌的 127.0.0.1 /api/*。 +const TOKEN = new URLSearchParams(location.search).get("t") || ""; +const $ = (id) => document.getElementById(id); +const esc = (s) => String(s == null ? "" : s).replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c])); +const ic = (id) => ``; +const shortTime = (t) => { if (!t) return "—"; const s = String(t); return s.includes("T") ? s.replace("T", " ").slice(0, 16) : s.slice(0, 16); }; +const clone = (o) => JSON.parse(JSON.stringify(o)); + +// ---------- i18n ---------- +let LANG = localStorage.getItem("tb_lang") || "en"; +const T = { + "ribbon.local": ["Local console", "本机控制台"], + "ribbon.host": ["127.0.0.1 · only you can see this · not the shared library", "127.0.0.1 · 仅你自己可见 · 不是线上共享库"], + "ribbon.onlinePre": ["Online", "对应线上"], + "brand.name": ["Team Brain", "团队大脑"], + "nav.overview": ["Overview", "概览"], "nav.sessions": ["My Sessions", "我的 Session"], "nav.log": ["Activity", "活动日志"], "nav.config": ["Collection", "采集配置"], + "foot.collector": ["Collector", "采集常驻"], "foot.collectorRun": ["Collector · running", "采集常驻 · 运行中"], + "foot.sync": ["synced {t}", "同步 {t}"], "foot.noSync": ["no sync yet", "尚无同步"], + "who": ["{name}'s local footprint", "{name} 的本机足迹"], + "ov.title": ["Overview", "概览"], + "ov.sub": ["What this machine has published to the team's shared library — at a glance, retractable anytime.", "team-brain 从这台机器往全队共享库发布了什么 —— 一眼看清，随时可撤。"], + "ov.what": ["This page lives only on your machine (127.0.0.1) and only you can see it. It governs what the collector uploads from this computer to the team's shared library. Preview and exclude happen locally; only Retract reaches the shared library to delete an entry.", + "这个页面只在你本机（127.0.0.1）、只有你能看。它管的是「采集常驻从这台电脑上传到全队共享库的内容」。预览、排除都在本地发生；点「撤回」才会去线上库删掉对应那条。"], + "ov.recent": ["Recent activity", "最近活动"], + "ov.headCap": ["sessions have entered the team's shared library from this machine", "条 session 已从这台机器进入全队共享库"], + "ov.headPending": ["another {n} in debounce, uploading soon", "另有 {n} 条正在去抖、即将上传"], + "ov.headLocal": ["{n} in local ledger", "本机账本共 {n} 条"], + "ov.headSkip": ["{n} skipped / out of scope", "跳过 / 未在范围 {n} 条"], + "ov.recentEmpty": ["None yet.", "暂无。"], + "ov.countNote": ["Counts are reconstructed from sessions still on this machine, so they can be fewer than the shared library (which keeps everything ever uploaded, including sessions whose local files were since removed). The shared library is the source of truth.", + "计数是按本机现存的 session 重建的，可能比线上共享库少（共享库保留历来上传的全部，含本机源文件已删除的）。以线上共享库为准。"], + "ses.title": ["My Sessions", "我的 Session"], + "ses.sub": ["Conversations already in / about to enter the shared library. Click any to see exactly what teammates see.", "已进入 / 即将进入全队共享库的对话。点任意一条，看队友实际会看到的样子。"], + "th.status": ["Status", "状态"], "th.intent": ["Intent", "意图"], "th.tool": ["Tool", "工具"], "th.coord": ["Location", "在库位置"], "th.time": ["Worked", "干活时间"], + "ses.note": ["Only sessions in / entering the library are listed. Out-of-scope, skipped, or retracted ones are hidden.", "只列「在库 / 将入库」的。不在采集范围、被跳过、已撤回的不显示。"], + "ses.empty": ["No sessions in the shared library yet. New conversations appear here once they settle.", "还没有进入共享库的 session。新对话稳定后会自动出现在这里。"], + "badge.uploaded": ["In library", "已在库"], "badge.pending": ["Uploading", "即将上传"], + "log.title": ["Activity", "活动日志"], + "log.sub": ["What the collector has been doing recently — same source as `brain logs`. Idle ticks are hidden.", "采集常驻最近做了什么（和 brain logs 同源）。无事发生的空转已隐藏。"], + "log.empty": ["Nothing meaningful yet (collector just started / no uploads).", "还没有有意义的活动（常驻刚起 / 没传过东西）。"], + "cfg.title": ["Collection Settings", "采集配置"], + "cfg.sub": ["What's collected, where it goes, what's masked before upload. All changes affect this machine only.", "我在采什么、传到哪、上传前再抹掉什么。所有改动只影响这台机器。"], + "cfg.scope": ["Collection scope", "采集范围"], + "cfg.scopeWl": ["Whitelist only", "仅白名单目录"], "cfg.scopeWlDesc": ["Only sessions under the listed folders are uploaded; the rest stay on this machine.", "只有列出的目录下的 session 才会上传，其余留在本机。"], + "cfg.scopeAll": ["All sessions", "采集本机全部"], "cfg.scopeAllDesc": ["Every project's sessions are uploaded, including unlisted ones.", "所有项目的 session 都会传，含没列出的。"], + "cfg.noWhitelist": ["(no whitelist set)", "（未设置白名单）"], "cfg.addFolder": ["Add upload folder", "添加上传目录"], + "cfg.exclude": ["Excluded subtrees", "排除子树"], "cfg.excludeX": ["· not uploaded even within the whitelist", "· 白名单内也不传"], "cfg.none": ["(none)", "（无）"], "cfg.addExclude": ["Add excluded folder", "添加排除目录"], + "cfg.sources": ["Collection sources", "采集来源"], + "src.ccDesc": ["~/.claude/projects · main source", "~/.claude/projects · 主力来源"], "src.codexDesc": ["~/.codex/sessions", "~/.codex/sessions"], "src.traeDesc": ["~/.trae-cn/memory", "~/.trae-cn/memory"], "src.shDesc": ["session_history docs within the whitelist", "白名单内的 session_history 文档"], + "cfg.redact": ["Redaction", "脱敏"], "cfg.redactX": ["· masked before upload", "· 上传前抹除敏感内容"], + "cfg.builtinIntro": ["The following keys / credentials / paths are automatically masked to [REDACTED] before upload:", "以下密钥 / 凭据 / 路径上传前自动替换为 [REDACTED]："], + "cfg.wordlist": ["Personal redaction list · custom, local only", "个人脱敏词表 · 自定义、只存本机"], + "cfg.wordlistEmpty": ["Add customer names, your real name, internal code names — covering content-level things the built-in rules can't.", "把客户名、真名、内部代号加进来，补内置规则覆盖不到的内容级。"], + "cfg.addTerm": ["Add term / regex (/.../ for regex)", "添加词条 / 正则（/.../ 为正则）"], + "cfg.hitCount": ["{n} hits / {s} sessions", "本机命中 {n} 处 / {s} 条"], + "cfg.timing": ["Timing", "时机"], "cfg.interval": ["Scan interval", "扫描间隔"], "cfg.debounce": ["Debounce", "去抖"], + "cfg.unsaved": ["Unsaved changes …", "有未保存改动 …"], + "cfg.delta": ["Unsaved · this would add {a}, stop {s}", "有未保存改动 · 这么改会 多传 {a} 条、少传 {s} 条"], + "cfg.discard": ["Discard", "放弃"], "cfg.save": ["Save", "保存"], + "type.text": ["text", "文本"], "type.regex": ["regex", "正则"], + "dr.tabRed": ["Redacted (what teammates see)", "脱敏后（队友看到的）"], "dr.tabRaw": ["Local original", "本机原文"], + "dr.hintRetract": ["Retract = delete from the shared library and stop uploading (git history is still recoverable)", "撤回 = 从共享库删掉并不再上传（git 历史仍可考古）"], + "dr.hintExclude": ["Exclude = never upload this one", "排除 = 本机永不上传这条"], + "dr.retract": ["Retract from library", "从库中撤回"], "dr.exclude": ["Exclude (don't upload)", "排除（不上传）"], + "dr.truncated": ["· original truncated", "· 原文已截断"], "dr.loading": ["Loading…", "加载中…"], "dr.cantOpen": ["Can't open", "打不开"], "dr.loadFail": ["Load failed: {e}", "加载失败：{e}"], + "confirm.retract": ["Retract this from the team's shared library and add to the local exclude list?", "从全队共享库撤回这条，并加入本机排除名单？"], + "confirm.exclude": ["Add this to the local exclude list so it's never uploaded?", "把这条加入本机排除名单，永不上传？"], + "toast.retracted": ["Retracted and excluded", "已撤回并排除"], "toast.excludeOnly": ["Excluded", "已排除"], + "toast.retractPartial": ["Excluded (retract incomplete: {e})", "已排除（撤回未完成：{e}）"], + "toast.opFail": ["Action failed: {e}", "操作失败：{e}"], + "toast.termAdded": ["Added to personal redaction list (applies to new uploads)", "已加入个人脱敏词表（对新上传生效）"], + "toast.saved": ["Saved. Toggles apply immediately; folder-scope changes fully apply after restarting the collector.", "已保存。开关类即时生效；采集范围(目录)改动在 brain service restart 后完整生效。"], + "prompt.addFolder": ["Absolute path of folder to collect (~ ok)", "要采集的目录绝对路径（~ 也行）"], + "prompt.addExclude": ["Absolute path of folder to exclude", "要排除的目录绝对路径"], + "prompt.addTerm": ["Word / regex (use /pattern/flags for regex, e.g. /acme-\\w+/i)", "词 / 正则（用 /pattern/flags 表示正则，如 /acme-\\w+/i）"], + "connFail": ["Connection failed", "连接失败"], + "connFailSub": ["{e}. Reopen with `brain viewer` (the URL carries a local token).", "{e}。请用 brain viewer 重新打开（带本地 token 的地址）。"], +}; +const BUILTIN = { + en: ["API Key (sk-…)", "GitHub PAT", "GitLab token", "AWS", "Google", "Slack", "Stripe", "JWT", "Private key (PEM)", "password/token assignments", "credentials in URLs", "home paths → ~"], + zh: ["API Key (sk-…)", "GitHub PAT", "GitLab Token", "AWS", "Google", "Slack", "Stripe", "JWT", "私钥 (PEM)", "password/token 赋值", "URL 内账密", "家目录路径 → ~"], +}; +const L = () => (LANG === "zh" ? 1 : 0); +function t(key, vars) { let s = (T[key] && T[key][L()]) ?? key; if (vars) for (const k in vars) s = s.replaceAll("{" + k + "}", vars[k]); return s; } + +function applyStatic() { + document.documentElement.lang = LANG === "zh" ? "zh-CN" : "en"; + document.title = LANG === "zh" ? "团队大脑 · 本机控制台" : "Team Brain · Local Console"; + document.querySelectorAll("[data-i18n]").forEach((el) => { el.textContent = t(el.getAttribute("data-i18n")); }); + document.querySelectorAll("[data-i18n-html]").forEach((el) => { el.innerHTML = t(el.getAttribute("data-i18n-html")); }); + $("langtog").textContent = LANG === "zh" ? "EN" : "中文"; +} + +// ---------- API ---------- +async function api(p) { + const r = await fetch(p + (p.includes("?") ? "&" : "?") + "t=" + encodeURIComponent(TOKEN)); + if (!r.ok) { let e = {}; try { e = await r.json(); } catch {} throw new Error(e.error || ("HTTP " + r.status)); } + return r.json(); +} +async function apiPost(p, body) { + const r = await fetch(p + "?t=" + encodeURIComponent(TOKEN), { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body || {}) }); + if (!r.ok) { let e = {}; try { e = await r.json(); } catch {} throw new Error(e.error || ("HTTP " + r.status)); } + return r.json(); +} +let toastT; +function toast(msg) { const el = $("toast"); el.textContent = msg; el.classList.add("on"); clearTimeout(toastT); toastT = setTimeout(() => el.classList.remove("on"), 2800); } + +let SESSIONS = { uploaded: [], pending: [] }; +let CUR = null, CFG = null, WCFG = null, REDACT = { terms: [] }, OV = null; + +// ---------- overview ---------- +function renderOverview(d) { + OV = d; + $("sb-who").textContent = t("who", { name: d.me?.name || d.me?.id || (LANG === "zh" ? "本机" : "this machine") }); + $("sb-n").textContent = d.counts.uploaded + d.counts.pending; + $("sb-run").textContent = t("foot.collectorRun"); + $("sb-sync").textContent = d.lastSync ? t("foot.sync", { t: d.lastSync.slice(0, 19).replace("T", " ").slice(11) }) : t("foot.noSync"); + $("sb-ver").textContent = `${d.me?.id || ""} · ${d.version}`; + if (d.server_url) { const a = $("rb-online"); a.textContent = d.server_url.replace(/^https?:\/\//, ""); a.href = d.server_url; } + $("ov-headline").innerHTML = `
+ ${d.counts.uploaded} + ${t("ov.headCap")}
${t("ov.headPending", { n: d.counts.pending })} + ${t("ov.headLocal", { n: d.counts.localTotal })}
${t("ov.headSkip", { n: d.counts.skipped })}
`; +} + +// ---------- sessions ---------- +const badge = (st) => st === "pending" + ? `${ic("i-clock")}${t("badge.pending")}` + : `${ic("i-check")}${t("badge.uploaded")}`; +const rowHtml = (r) => `${badge(r.status)}${esc(r.intent)}${esc(r.tool)}${esc(r.coord)}${shortTime(r.time)}`; +function renderSessions() { + const rows = [...SESSIONS.pending, ...SESSIONS.uploaded]; + $("sb-n").textContent = rows.length; + $("rows").innerHTML = rows.length ? rows.map(rowHtml).join("") : `
${t("ses.empty")}
`; + const top = SESSIONS.uploaded.slice(0, 3); + $("ov-recent").innerHTML = top.length + ? `${top.map((r) => ``).join("")}
${shortTime(r.time)}${esc(r.intent)}${esc(r.coord)}
` + : `
${t("ov.recentEmpty")}
`; +} + +// ---------- log ---------- +let LOGLINES = []; +function renderLog(lines) { + LOGLINES = lines || []; + if (!LOGLINES.length) { $("logbox").innerHTML = `
${t("log.empty")}
`; return; } + $("logbox").innerHTML = LOGLINES.slice().reverse().map((l) => { + const m = l.match(/^(\S+)\s+(\w+)\s+([\s\S]*)$/); + if (!m) return `
${esc(l)}
`; + const [, ts, lv, msg] = m; + return `
${esc(ts.slice(0, 19).replace("T", " "))}${esc(lv)}${esc(msg)}
`; + }).join(""); +} + +// ---------- config ---------- +const dirty = () => WCFG && CFG && JSON.stringify(WCFG) !== JSON.stringify(CFG); +function renderConfig() { + if (!WCFG) return; + const wl = !WCFG.collect_all, folders = WCFG.upload_folders || [], excl = WCFG.exclude || []; + const tog = (key) => ``; + let h = `
${ic("i-folder")}${t("cfg.scope")}
+
+
${ic("i-folder")}${t("cfg.scopeWl")}
${t("cfg.scopeWlDesc")}
+
${ic("i-layers")}${t("cfg.scopeAll")}
${t("cfg.scopeAllDesc")}
+
+
${folders.length ? folders.map((f, i) => `
${ic("i-folder")}${esc(f)}
`).join("") : `
${t("cfg.noWhitelist")}
`}
${ic("i-plus")}${t("cfg.addFolder")}
`; + h += `
${ic("i-ban")}${t("cfg.exclude")}${t("cfg.excludeX")}
${excl.length ? excl.map((f, i) => `
${ic("i-ban")}${esc(f)}
`).join("") : `
${t("cfg.none")}
`}
${ic("i-plus")}${t("cfg.addExclude")}
`; + h += `
${ic("i-layers")}${t("cfg.sources")}
+
Claude Code${t("src.ccDesc")}
+
Codex${t("src.codexDesc")}
${tog("codex")}
+
Trae memory${t("src.traeDesc")}
${tog("trae_memory")}
+
session_history *.md${t("src.shDesc")}
${tog("session_history_md")}
`; + h += `
${ic("i-shield")}${t("cfg.redact")}${t("cfg.redactX")}
+
${t("cfg.builtinIntro")}
+
${BUILTIN[LANG === "zh" ? "zh" : "en"].map((x) => `${ic("i-check")}${esc(x)}`).join("")}
+
${t("cfg.wordlist")}
+
${REDACT.terms.length ? REDACT.terms.map((x) => `
${ic("i-shield")}${esc(x.pattern)}${t(x.type === "regex" ? "type.regex" : "type.text")}
${t("cfg.hitCount", { n: x.count, s: x.sessions })}
`).join("") : `
${t("cfg.wordlistEmpty")}
`}
${ic("i-plus")}${t("cfg.addTerm")}
`; + h += `
${ic("i-clock")}${t("cfg.timing")}
+
${t("cfg.interval")}
${WCFG.interval_sec}s
+
${t("cfg.debounce")}
${WCFG.debounce_sec}s
`; + if (dirty()) h += `
${ic("i-alert")}${t("cfg.unsaved")}
`; + $("cfg").innerHTML = h; + if (dirty()) refreshDelta(); +} +let deltaT; +function refreshDelta() { + clearTimeout(deltaT); + deltaT = setTimeout(async () => { + try { const d = await apiPost("/api/config-dryrun", WCFG); const el = $("delta"); if (el) el.innerHTML = t("cfg.delta", { a: d.willAdd, s: d.willStop }); } catch {} + }, 250); +} +async function saveConfig() { try { await apiPost("/api/config", WCFG); CFG = clone(WCFG); renderConfig(); toast(t("toast.saved")); } catch (e) { toast(t("toast.opFail", { e: e.message })); } } +async function refreshRedact() { try { REDACT = await api("/api/redact"); } catch {} } + +$("config").addEventListener("click", async (e) => { + const el = e.target.closest("[data-act]"); if (!el) return; + const act = el.dataset.act; + if (act === "scope") { WCFG.collect_all = el.dataset.val === "all"; renderConfig(); } + else if (act === "toggle") { const k = el.dataset.key; WCFG[k] = WCFG[k] === false; renderConfig(); } + else if (act === "addfolder") { const p = prompt(t("prompt.addFolder")); if (p) { (WCFG.upload_folders ||= []).push(p.trim()); renderConfig(); } } + else if (act === "rmfolder") { WCFG.upload_folders.splice(+el.dataset.i, 1); renderConfig(); } + else if (act === "addexcl") { const p = prompt(t("prompt.addExclude")); if (p) { (WCFG.exclude ||= []).push(p.trim()); renderConfig(); } } + else if (act === "rmexcl") { WCFG.exclude.splice(+el.dataset.i, 1); renderConfig(); } + else if (act === "save") saveConfig(); + else if (act === "discard") { WCFG = clone(CFG); renderConfig(); } + else if (act === "addterm") { const p = prompt(t("prompt.addTerm")); if (p) { const v = p.trim(); const type = /^\/.*\/[gimsuy]*$/.test(v) ? "regex" : "text"; await apiPost("/api/redact-add", { pattern: v, type }); await refreshRedact(); renderConfig(); toast(t("toast.termAdded")); } } + else if (act === "rmterm") { await apiPost("/api/redact-remove", { pattern: el.dataset.p }); await refreshRedact(); renderConfig(); } +}); + +// ---------- drawer ---------- +function setTab(tab) { + document.querySelectorAll(".dtab").forEach((x) => x.classList.toggle("on", x.dataset.tab === tab)); + $("d-body").textContent = CUR ? (tab === "red" ? CUR.redacted : CUR.raw) : ""; +} +async function openDetail(file) { + $("scrim").classList.add("on"); $("drawer").classList.add("on"); + $("d-title").textContent = t("dr.loading"); $("d-coord").textContent = ""; $("d-body").textContent = ""; + $("d-exclude").style.display = "none"; $("d-hint").textContent = ""; CUR = null; + let d; try { d = await api("/api/session?file=" + encodeURIComponent(file)); } catch (e) { $("d-body").textContent = t("dr.loadFail", { e: e.message }); return; } + if (d.error) { $("d-title").textContent = t("dr.cantOpen"); $("d-coord").innerHTML = d.coord ? `${esc(d.coord)}` : ""; $("d-body").textContent = d.error; return; } + CUR = d; CUR.file = file; + $("d-title").textContent = d.intent || d.id; + $("d-coord").innerHTML = `${esc(d.coord || "")} · ${esc(d.tool)}${d.truncated ? " " + t("dr.truncated") : ""}`; + const up = d.status === "uploaded"; + $("d-exclude-t").textContent = up ? t("dr.retract") : t("dr.exclude"); + $("d-hint").textContent = up ? t("dr.hintRetract") : t("dr.hintExclude"); + $("d-exclude").style.display = "inline-flex"; + setTab("red"); +} +function closeDrawer() { $("scrim").classList.remove("on"); $("drawer").classList.remove("on"); } +$("d-exclude").onclick = async () => { + if (!CUR) return; + const up = CUR.status === "uploaded"; + if (!confirm(up ? t("confirm.retract") : t("confirm.exclude"))) return; + try { + const r = await apiPost("/api/exclude", { file: CUR.file }); + toast(up ? (r.retract?.ok ? t("toast.retracted") : t("toast.retractPartial", { e: r.retract?.error || "?" })) : t("toast.excludeOnly")); + closeDrawer(); await reload(); + } catch (e) { toast(t("toast.opFail", { e: e.message })); } +}; + +// ---------- nav / events ---------- +function go(id) { + document.querySelectorAll(".nav a").forEach((x) => x.classList.toggle("on", x.dataset.go === id)); + document.querySelectorAll(".page").forEach((p) => p.classList.toggle("on", p.id === id)); +} +document.querySelectorAll(".nav a").forEach((a) => a.onclick = () => go(a.dataset.go)); +document.querySelectorAll(".dtab").forEach((x) => x.onclick = () => setTab(x.dataset.tab)); +$("d-close").onclick = closeDrawer; +$("scrim").onclick = closeDrawer; +$("langtog").onclick = () => { + LANG = LANG === "zh" ? "en" : "zh"; localStorage.setItem("tb_lang", LANG); + applyStatic(); + if (OV) renderOverview(OV); renderSessions(); renderLog(LOGLINES); renderConfig(); +}; +document.addEventListener("keydown", (e) => { if (e.key === "Escape") closeDrawer(); }); +document.addEventListener("click", (e) => { if (e.target.closest("[data-act]")) return; const tr = e.target.closest("[data-file]"); if (tr) openDetail(tr.getAttribute("data-file")); }); + +async function reload() { + const [ov, ss] = await Promise.all([api("/api/overview"), api("/api/sessions")]); + renderOverview(ov); SESSIONS = ss; renderSessions(); +} +async function init() { + applyStatic(); + try { + const [ov, ss, lg, rd] = await Promise.all([api("/api/overview"), api("/api/sessions"), api("/api/log"), api("/api/redact")]); + REDACT = rd; CFG = ov.config; WCFG = clone(CFG); + renderOverview(ov); SESSIONS = ss; renderSessions(); renderLog(lg.lines); renderConfig(); + } catch (e) { + document.querySelector(".main").innerHTML = `

${t("connFail")}

${esc(t("connFailSub", { e: e.message }))}

`; + } +} +init();