diff --git a/src/log-mask.ts b/src/log-mask.ts index 4792109b1f..fa59d479f5 100644 --- a/src/log-mask.ts +++ b/src/log-mask.ts @@ -10,6 +10,7 @@ */ import { createHash } from "node:crypto"; +import os from "node:os"; const PRIVATE_HOST = ""; @@ -137,6 +138,41 @@ export function maskUrlsForPublish(text: string): string { return text.replace(/https?:\/\/[^\s"'<>]+/gi, (m) => maskUrlForPublish(m)); } +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** #4427: strip the running user's home directory out of absolute paths in + * publish-boundary text, rewriting it to `~` (the shell universal for $HOME). + * This is the identity leak #3959 proved in the wild — a public diagnostics + * report naming /home//... hands a stranger the reporter's username and + * home layout. Only THIS machine's os.homedir() is rewritten, as an ANCHORED + * match on BOTH flanks: preceded by start-of-string or a non-token boundary + * (whitespace / quote / bracket / colon / equals) AND followed by a path + * separator (/ or \), a quote, whitespace or end-of-string — so sibling homes + * (/home/), URL hosts and URL pathnames are all left alone. The result + * stays actionable: ~/.cache/billion-context/... is exactly where the user + * looks to recover. Idempotent (~ has no homedir in it). */ +export function maskPathsForPublish(text: string): string { + const home = os.homedir(); + if (!text || !home) return text; + const re = new RegExp("(? l.includes("[storage]")).slice(-STORAGE_LOG_TAIL_LINES); + return lines + .filter((l) => l.includes("[storage]")) + .slice(-STORAGE_LOG_TAIL_LINES) + .map((l) => { + const masked = redactForPublish(l); + return masked.length > LOG_LINE_MAX_CHARS ? masked.slice(0, LOG_LINE_MAX_CHARS - 1) + "…" : masked; + }); } // #3611: the evidence chain of a dsh native-compaction episode, exactly what a @@ -126,11 +141,10 @@ function conflictLogTail(lines: string[]): string[] { .filter((l) => CONFLICT_LOG_PATTERNS.some((p) => l.includes(p))) .slice(-CONFLICT_LOG_TAIL_LINES) .map((l) => { - // #3811: redactSecretsInText scrubs credential shapes only — a URL in a - // log line keeps its host. maskUrlsForPublish masks it unconditionally, - // so under BILI_LOG_MASK_HOSTS=0 (real hosts in the on-disk log) the - // published tail still cannot leak them. - const masked = maskUrlsForPublish(redactSecretsInText(l)); + // #3811: redactForPublish scrubs credential shapes AND masks URL hosts + // unconditionally, so under BILI_LOG_MASK_HOSTS=0 (real hosts in the + // on-disk log) the published tail still cannot leak them. + const masked = redactForPublish(l); return masked.length > LOG_LINE_MAX_CHARS ? masked.slice(0, LOG_LINE_MAX_CHARS - 1) + "…" : masked; }); } @@ -242,11 +256,12 @@ function configSectionLines(): string[] { } let text: string; try { - // #3811: maskUrlsForPublish is the outermost sink net — it masks every - // non-public endpoint host whether it appears as a VALUE (upstream/proxy) - // or a providers-table KEY, and drops userinfo/query/hash (subsuming the - // old userinfo-only strip). Always on: the blob is publish-boundary. - text = maskUrlsForPublish(redactSecretsInText(JSON.stringify(maskSecretValues(parsed)))); + // #3811: redactForPublish is the outermost sink net for this section — it + // masks every non-public endpoint host whether it appears as a VALUE + // (upstream/proxy) or a providers-table KEY, drops userinfo/query/hash + // (subsuming the old userinfo-only strip), AND scrubs credential shapes. + // Always on: the blob is publish-boundary. + text = redactForPublish(JSON.stringify(maskSecretValues(parsed))); } catch { lines.push("(config could not be serialized)"); return lines; @@ -347,7 +362,16 @@ export function buildDiagnostics(topic?: string): DiagnosticsReport { if (def) lines.push(...def.section(ctx)); lines.push(...hostClientVersionLines()); lines.push(...configSectionLines()); - const blob = lines.join("\n"); + // #4427: sink-level publish net — the blob lands in a PUBLIC GitHub issue + // body (copied verbatim AND embedded in the prefilled URL), so EVERY byte of + // every topic passes unconditional host masking + credential scrubbing right + // before serialization. This is the real choke point the header promises: it + // stops relying on each topic author remembering to sanitize its own fields + // (the storage section's raw log tail / free-form reason strings are exactly + // what slipped through before). Idempotent over the per-section redaction + // already applied upstream; both `blob` and the URL body derive from this one + // string so they can never diverge. + const blob = redactForPublish(lines.join("\n")); const title = def ? def.title() : `bili diagnostics report (v${VERSION}, node ${process.version}, ${process.platform}-${process.arch})`; diff --git a/tests/web-storage-diagnostics-redaction.test.ts b/tests/web-storage-diagnostics-redaction.test.ts new file mode 100644 index 0000000000..eb8ef03b1d --- /dev/null +++ b/tests/web-storage-diagnostics-redaction.test.ts @@ -0,0 +1,171 @@ +// #4427: the storage-legacy-only diagnostics blob lands in a PUBLIC GitHub issue +// body, but its storage section copied raw [storage] bytes (and free-form reason +// strings + the absolute failure-marker path) with NO publish-time redaction — +// unlike the config and conflict-log sections. These tests pin that EVERY byte of +// the storage report now passes the same unconditional publish-boundary net as +// the rest: credential shapes scrubbed, non-public URL hosts masked, and user- +// identifying $HOME prefixes rewritten to ~ — regardless of the local +// BILI_LOG_MASK_HOSTS preference, and equivalently in the copied blob AND the +// prefilled issue body. +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import * as path from "node:path"; +import { buildDiagnostics } from "../src/web/diagnostics.js"; +import { isMaskHostsEnabled, maskPathsForPublish, redactForPublish, setMaskHostsEnabled } from "../src/log-mask.js"; +import { bootstrapFailureMarkerPath } from "../src/storage/engine-bootstrap.js"; + +const ENV_KEYS = ["BILI_CONFIG_FILE", "DSH_HOME", "XDG_STATE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"] as const; +type EnvSnap = Record; + +function saveEnv(): EnvSnap { + const snap: EnvSnap = {}; + for (const k of ENV_KEYS) snap[k] = process.env[k]; + return snap; +} + +function restoreEnv(snap: EnvSnap): void { + for (const k of ENV_KEYS) { + if (snap[k] === undefined) delete process.env[k]; + else process.env[k] = snap[k]; + } +} + +/** Pin every path-relevant env var (unset ones deleted so homedir fallbacks can + * never reach the real machine's dirs) AND $HOME (so os.homedir() is + * deterministic); all restored in finally. */ +function withEnv(home: string | undefined, overrides: Record, fn: () => T): T { + const saved = saveEnv(); + const savedHome = process.env.HOME; + if (home !== undefined) process.env.HOME = home; + for (const k of ENV_KEYS) { + if (overrides[k] !== undefined) process.env[k] = overrides[k]; + else delete process.env[k]; + } + try { + return fn(); + } finally { + restoreEnv(saved); + if (savedHome === undefined) delete process.env.HOME; + else process.env.HOME = savedHome; + } +} + +test("#4427 maskPathsForPublish rewrites this machine's $HOME to ~, anchored on both flanks", () => { + const savedHome = process.env.HOME; + process.env.HOME = "/home/alice"; + try { + assert.equal(maskPathsForPublish("/home/alice/.local/share/billion-context/x.db"), "~/.local/share/billion-context/x.db"); + assert.equal(maskPathsForPublish("open failed at /home/alice/db"), "open failed at ~/db"); + assert.equal(maskPathsForPublish('ENOENT: no such file or directory, open \'/home/alice/index.db\''), "ENOENT: no such file or directory, open '~/index.db'"); + // sibling home: the char after /home/alice is 'x' (a token char), not a separator → untouched + assert.equal(maskPathsForPublish("/home/alicex/.local"), "/home/alicex/.local", "sibling home left alone"); + // URL pathname: /home/alice sits right after the host token (preceded by '/') → untouched here + // (host masking is a separate net; the path masker must not corrupt URLs) + assert.equal(maskPathsForPublish("https://relay.internal/home/alice/doc.pdf"), "https://relay.internal/home/alice/doc.pdf", "URL pathname not rewritten"); + assert.equal(maskPathsForPublish("~/.already-tilded"), "~/.already-tilded", "idempotent on already-tilded text"); + } finally { + if (savedHome === undefined) delete process.env.HOME; + else process.env.HOME = savedHome; + } +}); + +test("#4427 redactForPublish composes secret scrub + host mask + home-path strip in one pass", () => { + const savedHome = process.env.HOME; + process.env.HOME = "/home/bob"; + try { + const out = redactForPublish("probe https://user:SecretPass@db.internal:3306/app?token=FAKE_TOKEN at /home/bob/.cache/x authorization: bearer sk-live-abcdef1234567890 failed"); + assert.ok(!out.includes("db.internal"), "private URL host masked"); + assert.ok(!out.includes("SecretPass"), "URL userinfo password dropped"); + assert.ok(!out.includes("FAKE_TOKEN"), "URL query credential dropped"); + assert.ok(out.includes(""), "masked-host placeholder present"); + assert.ok(!out.includes("/home/bob"), "home dir stripped from absolute path"); + assert.ok(out.includes("~/.cache/x"), "path stays actionable via ~"); + assert.ok(!out.includes("sk-live-abcdef1234567890"), "bearer token scrubbed"); + assert.ok(out.includes(" { + const root = mkdtempSync(path.join(os.tmpdir(), "bili-diag-store-root-")); + const home = mkdtempSync(path.join(os.tmpdir(), "bili-diag-store-home-")); + const stateHome = path.join(root, "state"); + mkdirSync(path.join(stateHome, "billion-context"), { recursive: true }); + const logFile = path.join(stateHome, "billion-context", "bili.log"); + const longLine = "[storage] " + "E".repeat(300); + writeFileSync(logFile, [ + `[storage] sidecar probe https://demo:FAKE_PASSWORD@db.internal:3306/app?token=FAKE_TOKEN failed`, + `[storage] upstream 401 authorization: bearer sk-live-abcdef1234567890 rejected`, + `[storage] open failed at ${home}/.local/share/billion-context/index.db (EACCES permission denied)`, + longLine, + "[storage] index.db healthy (sqlite unified)", + ].join("\n") + "\n"); + const env = { DSH_HOME: path.join(root, "none"), XDG_STATE_HOME: stateHome, XDG_CONFIG_HOME: root, XDG_DATA_HOME: root, XDG_CACHE_HOME: root, BILI_CONFIG_FILE: path.join(root, "absent.json") }; + try { + const rep = withEnv(home, env, () => buildDiagnostics("storage-legacy-only")); + const blob = rep.blob; + + assert.ok(blob.includes("--- storage ---"), "storage section present"); + assert.ok(blob.includes("[storage] index.db healthy (sqlite unified)"), "benign [storage] line survives verbatim"); + + assert.ok(!blob.includes("db.internal"), "private relay host in a [storage] line masked"); + assert.ok(!blob.includes("FAKE_PASSWORD"), "URL userinfo password dropped"); + assert.ok(!blob.includes("token=FAKE_TOKEN"), "URL query credential dropped"); + assert.ok(!blob.includes("sk-live-abcdef1234567890"), "bearer token in a [storage] line scrubbed"); + assert.ok(blob.includes(" { + const root = mkdtempSync(path.join(os.tmpdir(), "bili-diag-store-off-")); + const stateHome = path.join(root, "state"); + mkdirSync(path.join(stateHome, "billion-context"), { recursive: true }); + writeFileSync(path.join(stateHome, "billion-context", "bili.log"), + "[storage] routed call identified via https://corp-relay.internal.corp/v1/model (ECONNRESET)\n"); + const env = { DSH_HOME: path.join(root, "none"), XDG_STATE_HOME: stateHome, XDG_CONFIG_HOME: root, XDG_DATA_HOME: root, XDG_CACHE_HOME: root, BILI_CONFIG_FILE: path.join(root, "absent.json") }; + const prev = isMaskHostsEnabled(); + try { + setMaskHostsEnabled(false); // operator opted out of LOCAL log host-masking + const blob = withEnv(undefined, env, () => buildDiagnostics("storage-legacy-only")).blob; + assert.ok(!blob.includes("corp-relay.internal.corp"), "private host masked in PUBLISHED blob despite local switch OFF"); + assert.ok(blob.includes(""), "masked-host placeholder present under switch-OFF"); + } finally { + setMaskHostsEnabled(prev); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("#4427 force-retry marker path keeps actionability while shedding the username", () => { + const home = mkdtempSync(path.join(os.tmpdir(), "bili-diag-marker-home-")); + try { + // cache root under $HOME → the absolute marker carries the user's home + // prefix; both the resolve and the redaction must run under the same + // pinned $HOME (os.homedir() is read live inside redactForPublish). + const published = withEnv(home, { XDG_CACHE_HOME: path.join(home, ".cache") }, () => { + const abs = bootstrapFailureMarkerPath(); + assert.ok(abs.startsWith(`${home}${path.sep}.cache${path.sep}billion-context`), "marker resolves under the cache root"); + return redactForPublish(abs); + }); + assert.equal(published, `~${path.sep}.cache${path.sep}billion-context${path.sep}sqlite${path.sep}.bootstrap-failed.json`, "published marker is ~-anchored, not username-bearing"); + assert.ok(!published.includes(home), "no absolute home dir survives into the public report"); + } finally { + rmSync(home, { recursive: true, force: true }); + } +});