Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions src/log-mask.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/

import { createHash } from "node:crypto";
import os from "node:os";

const PRIVATE_HOST = "<private-host>";

Expand Down Expand Up @@ -137,6 +138,41 @@ export function maskUrlsForPublish(text: string): string {
return text.replace(/https?:\/\/[^\s"'<>]+/gi, (m) => maskUrlForPublish(m));
}

function escapeRegExp(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

/** #4427: strip the running user's home directory out of absolute paths in
* publish-boundary text, rewriting it to `~` (the shell universal for $HOME).
* This is the identity leak #3959 proved in the wild — a public diagnostics
* report naming /home/<user>/... hands a stranger the reporter's username and
* home layout. Only THIS machine's os.homedir() is rewritten, as an ANCHORED
* match on BOTH flanks: preceded by start-of-string or a non-token boundary
* (whitespace / quote / bracket / colon / equals) AND followed by a path
* separator (/ or \), a quote, whitespace or end-of-string — so sibling homes
* (/home/<other>), URL hosts and URL pathnames are all left alone. The result
* stays actionable: ~/.cache/billion-context/... is exactly where the user
* looks to recover. Idempotent (~ has no homedir in it). */
export function maskPathsForPublish(text: string): string {
const home = os.homedir();
if (!text || !home) return text;
const re = new RegExp("(?<![/\\w.\\-])" + escapeRegExp(home) + "(?=[/\\\\\"'\\s]|$)", "g");
return text.replace(re, "~");
}

/** #4427: the single publish-boundary redaction primitive for any byte that
* lands in a PUBLIC artifact (the diagnostics blob / prefilled issue body) —
* credential-shape scrubbing, then UNCONDITIONAL non-public URL-host masking,
* then home-dir path stripping. Composes the three existing nets in one fixed
* order so every caller applies them instead of re-deriving it; idempotent, so
* a second pass over already-redacted text is a no-op (the sink-level net in
* buildDiagnostics re-runs this over per-section output safely). Never gated by
* BILI_LOG_MASK_HOSTS — the local log host-visibility preference must not
* weaken the publish boundary (#3811). */
export function redactForPublish(text: string): string {
return maskPathsForPublish(maskUrlsForPublish(redactSecretsInText(text)));
}

/** Mask a URL for DIAGNOSTIC payloads (#3651): same public/private rule as
* maskUrlForLog, with a LOOPBACK EXEMPTION — localhost / 127.x / ::1 /
* 0.0.0.0 identify bili's own local proxy, never an external party (same
Expand Down
4 changes: 3 additions & 1 deletion src/storage/engine-bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,9 @@ export interface CooldownMarker {

/** Absolute path of the failure marker (#3788): surfaced in the storage
* diagnostics blob so a user can delete it to force an immediate retry
* instead of waiting out the whole cooldown window. */
* instead of waiting out the whole cooldown window. #4427: the publish-boundary
* sink (redactForPublish) rewrites any $HOME prefix to ~ before the blob ships,
* so this absolute form never leaks the user's home dir / username publicly. */
export function bootstrapFailureMarkerPath(): string {
return cooldownFile();
}
Expand Down
66 changes: 45 additions & 21 deletions src/web/diagnostics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,18 @@
* `--- recent conflict-related log lines ---` (the exact signatures a
* maintainer greps to adjudicate a native-compaction episode).
*
* PUBLISH DISCIPLINE: the blob lands in a PUBLIC GitHub issue body. Secret
* values are replaced before serialization and redactSecretsInText is applied
* as a sink-level net; only profile NAMES are included, never absolute paths
* (they would leak usernames). #3811: non-public endpoint hosts are masked too —
* config URLs and conflict-log lines run through maskUrlsForPublish, which masks
* unconditionally so the blob stays host-safe even when BILI_LOG_MASK_HOSTS=0
* (the local host-visibility preference must not weaken the publish boundary).
* PUBLISH DISCIPLINE: the blob lands in a PUBLIC GitHub issue body (copied
* verbatim AND embedded in the prefilled URL). Every byte passes redactForPublish
* at the SINK (buildDiagnostics) — unconditional host masking + credential-shape
* scrubbing, ignoring BILI_LOG_MASK_HOSTS so the local host-visibility preference
* can never weaken the publish boundary (#3811/#4427). Per-section redaction
* (config values, log tails, storage reason strings) runs upstream too; the sink
* is the guarantee that NO topic — present or future — can leak a private host, a
* credential shape, or a user-identifying home path (redactForPublish also rewrites
* $HOME→~ across the whole blob) even if its author forgets. Only profile NAMES are
* included; the storage force-retry lever keeps its absolute marker path for
* actionability, but the sink ~-normalizes any $HOME prefix so no username reaches
* the public body (#4427).
*
* Contract: NEVER throws — every fs/listSessions access is guarded so a
* broken log file or an unbuilt store degrades the blob, not the endpoint.
Expand All @@ -37,7 +42,7 @@ import { snapshotShadowStatusWire } from "../server/snapshot-render.js";
import { dshProfileDirs } from "../dsh-channel.js";
import { resolveDshHome } from "../client-config.js";
import { safeReadJson } from "../config.js";
import { maskUrlsForPublish, redactSecretsInText } from "../log-mask.js";
import { redactForPublish } from "../log-mask.js";
import { getLogPath } from "../logger.js";
import { configFile, defaultLogFile } from "../paths.js";
import { listSessions } from "../session.js";
Expand Down Expand Up @@ -103,9 +108,19 @@ function candidateLogLines(): string[] {
}

/** Last N `[storage]` lines from the pre-fetched candidate set ([] when none
* match). The caller reads the rotated pair once per report (#3813). */
* match). The caller reads the rotated pair once per report (#3813). #4427:
* these are VERBATIM on-disk bytes (written under whatever BILI_LOG_MASK_HOSTS
* was in effect), so each line gets the same publish-boundary redaction +
* length bound as conflictLogTail — a [storage] line can carry a URL with
* credentials, a private host, or an absolute path. */
function storageLogTail(lines: string[]): string[] {
return lines.filter((l) => l.includes("[storage]")).slice(-STORAGE_LOG_TAIL_LINES);
return lines
.filter((l) => l.includes("[storage]"))
.slice(-STORAGE_LOG_TAIL_LINES)
.map((l) => {
const masked = redactForPublish(l);
return masked.length > LOG_LINE_MAX_CHARS ? masked.slice(0, LOG_LINE_MAX_CHARS - 1) + "…" : masked;
});
}

// #3611: the evidence chain of a dsh native-compaction episode, exactly what a
Expand All @@ -126,11 +141,10 @@ function conflictLogTail(lines: string[]): string[] {
.filter((l) => CONFLICT_LOG_PATTERNS.some((p) => l.includes(p)))
.slice(-CONFLICT_LOG_TAIL_LINES)
.map((l) => {
// #3811: redactSecretsInText scrubs credential shapes only — a URL in a
// log line keeps its host. maskUrlsForPublish masks it unconditionally,
// so under BILI_LOG_MASK_HOSTS=0 (real hosts in the on-disk log) the
// published tail still cannot leak them.
const masked = maskUrlsForPublish(redactSecretsInText(l));
// #3811: redactForPublish scrubs credential shapes AND masks URL hosts
// unconditionally, so under BILI_LOG_MASK_HOSTS=0 (real hosts in the
// on-disk log) the published tail still cannot leak them.
const masked = redactForPublish(l);
return masked.length > LOG_LINE_MAX_CHARS ? masked.slice(0, LOG_LINE_MAX_CHARS - 1) + "…" : masked;
});
}
Expand Down Expand Up @@ -242,11 +256,12 @@ function configSectionLines(): string[] {
}
let text: string;
try {
// #3811: maskUrlsForPublish is the outermost sink net — it masks every
// non-public endpoint host whether it appears as a VALUE (upstream/proxy)
// or a providers-table KEY, and drops userinfo/query/hash (subsuming the
// old userinfo-only strip). Always on: the blob is publish-boundary.
text = maskUrlsForPublish(redactSecretsInText(JSON.stringify(maskSecretValues(parsed))));
// #3811: redactForPublish is the outermost sink net for this section — it
// masks every non-public endpoint host whether it appears as a VALUE
// (upstream/proxy) or a providers-table KEY, drops userinfo/query/hash
// (subsuming the old userinfo-only strip), AND scrubs credential shapes.
// Always on: the blob is publish-boundary.
text = redactForPublish(JSON.stringify(maskSecretValues(parsed)));
} catch {
lines.push("(config could not be serialized)");
return lines;
Expand Down Expand Up @@ -347,7 +362,16 @@ export function buildDiagnostics(topic?: string): DiagnosticsReport {
if (def) lines.push(...def.section(ctx));
lines.push(...hostClientVersionLines());
lines.push(...configSectionLines());
const blob = lines.join("\n");
// #4427: sink-level publish net — the blob lands in a PUBLIC GitHub issue
// body (copied verbatim AND embedded in the prefilled URL), so EVERY byte of
// every topic passes unconditional host masking + credential scrubbing right
// before serialization. This is the real choke point the header promises: it
// stops relying on each topic author remembering to sanitize its own fields
// (the storage section's raw log tail / free-form reason strings are exactly
// what slipped through before). Idempotent over the per-section redaction
// already applied upstream; both `blob` and the URL body derive from this one
// string so they can never diverge.
const blob = redactForPublish(lines.join("\n"));
const title = def
? def.title()
: `bili diagnostics report (v${VERSION}, node ${process.version}, ${process.platform}-${process.arch})`;
Expand Down
171 changes: 171 additions & 0 deletions tests/web-storage-diagnostics-redaction.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
// #4427: the storage-legacy-only diagnostics blob lands in a PUBLIC GitHub issue
// body, but its storage section copied raw [storage] bytes (and free-form reason
// strings + the absolute failure-marker path) with NO publish-time redaction —
// unlike the config and conflict-log sections. These tests pin that EVERY byte of
// the storage report now passes the same unconditional publish-boundary net as
// the rest: credential shapes scrubbed, non-public URL hosts masked, and user-
// identifying $HOME prefixes rewritten to ~ — regardless of the local
// BILI_LOG_MASK_HOSTS preference, and equivalently in the copied blob AND the
// prefilled issue body.
import test from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import os from "node:os";
import * as path from "node:path";
import { buildDiagnostics } from "../src/web/diagnostics.js";
import { isMaskHostsEnabled, maskPathsForPublish, redactForPublish, setMaskHostsEnabled } from "../src/log-mask.js";
import { bootstrapFailureMarkerPath } from "../src/storage/engine-bootstrap.js";

const ENV_KEYS = ["BILI_CONFIG_FILE", "DSH_HOME", "XDG_STATE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"] as const;
type EnvSnap = Record<string, string | undefined>;

function saveEnv(): EnvSnap {
const snap: EnvSnap = {};
for (const k of ENV_KEYS) snap[k] = process.env[k];
return snap;
}

function restoreEnv(snap: EnvSnap): void {
for (const k of ENV_KEYS) {
if (snap[k] === undefined) delete process.env[k];
else process.env[k] = snap[k];
}
}

/** Pin every path-relevant env var (unset ones deleted so homedir fallbacks can
* never reach the real machine's dirs) AND $HOME (so os.homedir() is
* deterministic); all restored in finally. */
function withEnv<T>(home: string | undefined, overrides: Record<string, string | undefined>, fn: () => T): T {
const saved = saveEnv();
const savedHome = process.env.HOME;
if (home !== undefined) process.env.HOME = home;
for (const k of ENV_KEYS) {
if (overrides[k] !== undefined) process.env[k] = overrides[k];
else delete process.env[k];
}
try {
return fn();
} finally {
restoreEnv(saved);
if (savedHome === undefined) delete process.env.HOME;
else process.env.HOME = savedHome;
}
}

test("#4427 maskPathsForPublish rewrites this machine's $HOME to ~, anchored on both flanks", () => {
const savedHome = process.env.HOME;
process.env.HOME = "/home/alice";
try {
assert.equal(maskPathsForPublish("/home/alice/.local/share/billion-context/x.db"), "~/.local/share/billion-context/x.db");
assert.equal(maskPathsForPublish("open failed at /home/alice/db"), "open failed at ~/db");
assert.equal(maskPathsForPublish('ENOENT: no such file or directory, open \'/home/alice/index.db\''), "ENOENT: no such file or directory, open '~/index.db'");
// sibling home: the char after /home/alice is 'x' (a token char), not a separator → untouched
assert.equal(maskPathsForPublish("/home/alicex/.local"), "/home/alicex/.local", "sibling home left alone");
// URL pathname: /home/alice sits right after the host token (preceded by '/') → untouched here
// (host masking is a separate net; the path masker must not corrupt URLs)
assert.equal(maskPathsForPublish("https://relay.internal/home/alice/doc.pdf"), "https://relay.internal/home/alice/doc.pdf", "URL pathname not rewritten");
assert.equal(maskPathsForPublish("~/.already-tilded"), "~/.already-tilded", "idempotent on already-tilded text");
} finally {
if (savedHome === undefined) delete process.env.HOME;
else process.env.HOME = savedHome;
}
});

test("#4427 redactForPublish composes secret scrub + host mask + home-path strip in one pass", () => {
const savedHome = process.env.HOME;
process.env.HOME = "/home/bob";
try {
const out = redactForPublish("probe https://user:SecretPass@db.internal:3306/app?token=FAKE_TOKEN at /home/bob/.cache/x authorization: bearer sk-live-abcdef1234567890 failed");
assert.ok(!out.includes("db.internal"), "private URL host masked");
assert.ok(!out.includes("SecretPass"), "URL userinfo password dropped");
assert.ok(!out.includes("FAKE_TOKEN"), "URL query credential dropped");
assert.ok(out.includes("<private-host>"), "masked-host placeholder present");
assert.ok(!out.includes("/home/bob"), "home dir stripped from absolute path");
assert.ok(out.includes("~/.cache/x"), "path stays actionable via ~");
assert.ok(!out.includes("sk-live-abcdef1234567890"), "bearer token scrubbed");
assert.ok(out.includes("<masked"), "redaction marker visible");
} finally {
if (savedHome === undefined) delete process.env.HOME;
else process.env.HOME = savedHome;
}
});

test("#4427 storage-legacy-only blob leaks no credentials, private hosts, or home paths (unconditional)", () => {
const root = mkdtempSync(path.join(os.tmpdir(), "bili-diag-store-root-"));
const home = mkdtempSync(path.join(os.tmpdir(), "bili-diag-store-home-"));
const stateHome = path.join(root, "state");
mkdirSync(path.join(stateHome, "billion-context"), { recursive: true });
const logFile = path.join(stateHome, "billion-context", "bili.log");
const longLine = "[storage] " + "E".repeat(300);
writeFileSync(logFile, [
`[storage] sidecar probe https://demo:FAKE_PASSWORD@db.internal:3306/app?token=FAKE_TOKEN failed`,
`[storage] upstream 401 authorization: bearer sk-live-abcdef1234567890 rejected`,
`[storage] open failed at ${home}/.local/share/billion-context/index.db (EACCES permission denied)`,
longLine,
"[storage] index.db healthy (sqlite unified)",
].join("\n") + "\n");
const env = { DSH_HOME: path.join(root, "none"), XDG_STATE_HOME: stateHome, XDG_CONFIG_HOME: root, XDG_DATA_HOME: root, XDG_CACHE_HOME: root, BILI_CONFIG_FILE: path.join(root, "absent.json") };
try {
const rep = withEnv(home, env, () => buildDiagnostics("storage-legacy-only"));
const blob = rep.blob;

assert.ok(blob.includes("--- storage ---"), "storage section present");
assert.ok(blob.includes("[storage] index.db healthy (sqlite unified)"), "benign [storage] line survives verbatim");

assert.ok(!blob.includes("db.internal"), "private relay host in a [storage] line masked");
assert.ok(!blob.includes("FAKE_PASSWORD"), "URL userinfo password dropped");
assert.ok(!blob.includes("token=FAKE_TOKEN"), "URL query credential dropped");
assert.ok(!blob.includes("sk-live-abcdef1234567890"), "bearer token in a [storage] line scrubbed");
assert.ok(blob.includes("<masked"), "redaction marker visible in the tail");

assert.ok(!blob.includes(home), "no absolute home dir in the public blob (#3959 leak)");
assert.ok(blob.includes("~/.local/share/billion-context/index.db"), "home path preserved as ~ for actionability");

assert.ok(!blob.includes("E".repeat(300)), "overlong [storage] line truncated");
for (const l of blob.split("\n")) {
if (l.startsWith("[storage] ")) assert.ok(l.length <= 240, `published [storage] line bounded, got ${l.length}`);
}

assert.equal(new URL(rep.url).searchParams.get("body"), rep.blob, "issue body decodes back to the exact (redacted) blob");
} finally {
rmSync(root, { recursive: true, force: true });
rmSync(home, { recursive: true, force: true });
}
});

test("#4427 publish redaction holds even when local host-masking is disabled", () => {
const root = mkdtempSync(path.join(os.tmpdir(), "bili-diag-store-off-"));
const stateHome = path.join(root, "state");
mkdirSync(path.join(stateHome, "billion-context"), { recursive: true });
writeFileSync(path.join(stateHome, "billion-context", "bili.log"),
"[storage] routed call identified via https://corp-relay.internal.corp/v1/model (ECONNRESET)\n");
const env = { DSH_HOME: path.join(root, "none"), XDG_STATE_HOME: stateHome, XDG_CONFIG_HOME: root, XDG_DATA_HOME: root, XDG_CACHE_HOME: root, BILI_CONFIG_FILE: path.join(root, "absent.json") };
const prev = isMaskHostsEnabled();
try {
setMaskHostsEnabled(false); // operator opted out of LOCAL log host-masking
const blob = withEnv(undefined, env, () => buildDiagnostics("storage-legacy-only")).blob;
assert.ok(!blob.includes("corp-relay.internal.corp"), "private host masked in PUBLISHED blob despite local switch OFF");
assert.ok(blob.includes("<private-host>"), "masked-host placeholder present under switch-OFF");
} finally {
setMaskHostsEnabled(prev);
rmSync(root, { recursive: true, force: true });
}
});

test("#4427 force-retry marker path keeps actionability while shedding the username", () => {
const home = mkdtempSync(path.join(os.tmpdir(), "bili-diag-marker-home-"));
try {
// cache root under $HOME → the absolute marker carries the user's home
// prefix; both the resolve and the redaction must run under the same
// pinned $HOME (os.homedir() is read live inside redactForPublish).
const published = withEnv(home, { XDG_CACHE_HOME: path.join(home, ".cache") }, () => {
const abs = bootstrapFailureMarkerPath();
assert.ok(abs.startsWith(`${home}${path.sep}.cache${path.sep}billion-context`), "marker resolves under the cache root");
return redactForPublish(abs);
});
assert.equal(published, `~${path.sep}.cache${path.sep}billion-context${path.sep}sqlite${path.sep}.bootstrap-failed.json`, "published marker is ~-anchored, not username-bearing");
assert.ok(!published.includes(home), "no absolute home dir survives into the public report");
} finally {
rmSync(home, { recursive: true, force: true });
}
});
Loading