From 8b1749a197d69070c76b2bc442de9c226cb62244 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:42:12 +0800 Subject: [PATCH 1/5] fix: make upstream TLS trust failures self-diagnosable; add network.caFile (#3776) Security software doing HTTPS scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy re-signs the upstream with its own CA. Browsers trust it via the OS certificate store, but bili's own Node fetch reads only its built-in roots plus NODE_EXTRA_CA_CERTS, so these users hit SELF_SIGNED_CERT_IN_CHAIN, the upstream circuit opens after 3 pre-response failures, and the client only ever sees a generic 502 (#3773/#3776). - Trust-code TLS failures carry a concrete remediation hint on every channel rendering formatUpstreamError (log, 502 detail, circuit/alert detail) plus the panel tls alert banner (en+zh): export the scanner/vendor root to PEM -> NODE_EXTRA_CA_CERTS or network.caFile, or exclude node.exe from scanning. The trust-code set and remediation text live once in src/upstream-fail.ts; preflight reuses them. - New network.caFile / BILI_CA_FILE knob: a PEM file whose roots are MERGED into bili's own upstream fetch trust (direct + proxied dispatchers), never replacing built-ins or NODE_EXTRA_CA_CERTS. Unset = byte-identical behavior; dispatcher caches key on CA identity so edits apply on rotation. - The dispatch-catch 502 body gains an additive upstreamFailure{kind,code,hint} field when the failure is structured (upstream request failed / circuit open), so clients can self-diagnose without reading bili.log. Absent for everything else. Docs: CONFIGURATION.md en+zh. Tests: new hermetic suite (unit + in-process proxy e2e: untrusted MITM upstream => structured 502; same upstream trusted via caFile => 200); preflight transport-recovery assertion updated to the shared remediation. --- CONFIGURATION.md | 5 +- CONFIGURATION.zh-CN.md | 5 +- src/config.ts | 6 + src/fetch-util.ts | 14 +- src/knobs.ts | 13 + src/preflight.ts | 15 +- src/server.ts | 13 +- src/server/relay.ts | 18 +- src/upstream-alerts.ts | 9 + src/upstream-ca.ts | 73 ++++++ src/upstream-fail.ts | 52 ++++ src/upstream-proxy.ts | 43 +++- src/web/i18n.ts | 4 +- tests/issue3776-tls-mitm-diagnostics.test.ts | 245 +++++++++++++++++++ tests/preflight-transport-recovery.test.ts | 2 +- 15 files changed, 480 insertions(+), 37 deletions(-) create mode 100644 src/upstream-ca.ts create mode 100644 tests/issue3776-tls-mitm-diagnostics.test.ts diff --git a/CONFIGURATION.md b/CONFIGURATION.md index ecde8a6c95..da998ca57c 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -159,6 +159,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `network.decodedBodyBytes` | number | 10737418240 | BILI_DECODED_BODY_BYTES | Max DECODED bytes after content-encoding expansion (gzip/deflate/br/zstd), checked incrementally during decode (decode-stage 413). Default is 2x the receive budget, mirroring the legacy decode=2x-wire relationship; guards against decompression bombs. | | `network.outboundBodyBytes` | number | 5368709120 | BILI_OUTBOUND_BODY_BYTES | Max rebuilt request bytes forwarded upstream (forward-stage 413), counted as UTF-8 bytes of the final body string. Checked before the fetch so an over-budget rebuild never reaches the upstream watchdog allocation. | | `network.wsOutboundBytes` | number | 10485760 | BILI_WS_OUTBOUND_BYTES | Soft byte budget for the WebSocket lane only (#3701): when the final serialized Responses WS frame exceeds it, older images are shed from the OUTBOUND COPY first (archived as placeholder notes — originals stay in session storage, recoverable via read_original_image / decompress), then whatever remains is boundedly downscaled through the jimp maxDim ladder. If the frame still does not fit and the upstream closes with 1009 before any event, one strictly smaller resend is attempted within the same request. Set 0 to disable shedding (previous send-as-is behavior). HTTP lanes are unaffected — the hard refusal cap stays network.outboundBodyBytes. | +| `network.caFile` | string | unset | BILI_CA_FILE | Path to a PEM file with extra root CAs trusted by bili's OWN upstream fetches (#3776). Merged with Node's built-in roots and any `NODE_EXTRA_CA_CERTS` entries — never replaces them. For security software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy that re-signs the upstream: export that root CA to PEM and point this at it. Unset = default trust, byte-identical behavior. | | `guardrails` | object | {} (all defaults) | — | Guardrails: tool-output guardrails for the pi/omp plugin bundles (#3066) plus the proxy-side text-run stream guard (#3406). | | `guardrails.bashTimeout` | number | 60 | BILI_GUARDRAIL_BASH_TIMEOUT | Seconds injected as the bash tool's timeout when the model omits it (#3066 owner intent, re-affirmed #3366: an unsupervised bash call could hang a pi session for a whole day — the host schema provides no default). Set N to tune, 0 to disable injection (pi-native no-default). | | `guardrails.maxOutputBytes` | number | 50000 | BILI_GUARDRAIL_MAX_OUTPUT_BYTES | Combined text-output byte cap per tool result in the pi/omp plugin (0 disables the cap). Over-cap output is truncated head-first with a notice pointing at the host-saved full output. | @@ -883,7 +884,7 @@ Since #2030 every pure-behavior knob has a config-file key alongside its env var - **Type:** `{ upstreamTimeoutMs?: number; requestWatchdogMs?: number; replayRetryMax?: number; replayRetryBaseMs?: number; maxShrinkPerCompress?: number; keepAliveTimeoutMs?: number; clientErrorBackstopMs?: number; exposureLogIntervalMs?: number; streamKeepAliveMs?: number; preflightHoldMs?: number; preflightDeadEndCooldownMs?: number; proxyKeepAliveMaxMs?: number; postResponseLingerMs?: number; allowPrivateHttpProviders?: boolean }` - **Default:** `{}` (built-ins: `720000`, `2× upstream timeout`, `3`, `1500`, unset, `5000`, `30000`, `3600000`, `15000`, `30000`, `300000`, `55000`, `5000`, `false`) - **Status:** ACTIVE -- **Description:** Process-wide transport and retry timing (ms). Each key resolves env > file > default via its twin (`BILI_UPSTREAM_TIMEOUT_MS`, `BILI_REQUEST_WATCHDOG_MS`, `BILI_REPLAY_RETRY_MAX`, `BILI_REPLAY_RETRY_BASE_MS`, `BILI_MAX_SHRINK_PER_COMPRESS`, `BILI_KEEP_ALIVE_TIMEOUT_MS`, `BILI_CLIENT_ERROR_BACKSTOP_MS`, `BILI_EXPOSURE_LOG_INTERVAL_MS`, `BILI_STREAM_KEEPALIVE_MS`, `BILI_PREFLIGHT_HOLD_MS`, `BILI_PREFLIGHT_DEAD_END_COOLDOWN_MS`, `BILI_PROXY_KEEPALIVE_MAX_MS`, `BILI_POST_RESPONSE_LINGER_MS`, `BILI_ALLOW_PRIVATE_HTTP_PROVIDERS`). Semantics: `upstreamTimeoutMs` bounds time-to-first-byte and inter-chunk silence end-to-end (#551); `requestWatchdogMs` is the total-request budget (a negative value opts out); `replayRetryMax` counts attempts after transient upstream rejections (#189/#1688); `maxShrinkPerCompress` caps how much one compress may shrink the request before bili steers toward smaller ranges (#189); `streamKeepAliveMs` emits an SSE comment when the response has written zero bytes for that long (#1647); `proxyKeepAliveMaxMs` caps connection reuse through an upstream proxy below its recycle cadence (#1263), `0` uncaps; `postResponseLingerMs` is the graceful-close budget after a proxy-initiated post-response close — the socket is held waiting for the peer's FIN/TLS close_notify and destroyed at the deadline (`reason=linger-backstop`) (#1982); `allowPrivateHttpProviders` is an opt-in (#2984): when true, plain-HTTP provider baseUrls on private networks (RFC1918, 127/8, link-local, IPv6 ULA/link-local, `*.local`/`.lan`/`.internal`) are admitted by the agent-registry intake and summary-endpoint guards that otherwise trust loopback plaintext only — public IPs/domains and CGNAT (100.64/10) stay refused regardless, and the default `false` keeps pre-#2984 behavior byte-for-byte. Full details in the [env table](#config-file-keys-for-environment-knobs-2030). +- **Description:** Process-wide transport and retry timing (ms). Each key resolves env > file > default via its twin (`BILI_UPSTREAM_TIMEOUT_MS`, `BILI_REQUEST_WATCHDOG_MS`, `BILI_REPLAY_RETRY_MAX`, `BILI_REPLAY_RETRY_BASE_MS`, `BILI_MAX_SHRINK_PER_COMPRESS`, `BILI_KEEP_ALIVE_TIMEOUT_MS`, `BILI_CLIENT_ERROR_BACKSTOP_MS`, `BILI_EXPOSURE_LOG_INTERVAL_MS`, `BILI_STREAM_KEEPALIVE_MS`, `BILI_PREFLIGHT_HOLD_MS`, `BILI_PREFLIGHT_DEAD_END_COOLDOWN_MS`, `BILI_PROXY_KEEPALIVE_MAX_MS`, `BILI_POST_RESPONSE_LINGER_MS`, `BILI_ALLOW_PRIVATE_HTTP_PROVIDERS`, `BILI_CA_FILE`). Semantics: `upstreamTimeoutMs` bounds time-to-first-byte and inter-chunk silence end-to-end (#551); `requestWatchdogMs` is the total-request budget (a negative value opts out); `replayRetryMax` counts attempts after transient upstream rejections (#189/#1688); `maxShrinkPerCompress` caps how much one compress may shrink the request before bili steers toward smaller ranges (#189); `streamKeepAliveMs` emits an SSE comment when the response has written zero bytes for that long (#1647); `proxyKeepAliveMaxMs` caps connection reuse through an upstream proxy below its recycle cadence (#1263), `0` uncaps; `postResponseLingerMs` is the graceful-close budget after a proxy-initiated post-response close — the socket is held waiting for the peer's FIN/TLS close_notify and destroyed at the deadline (`reason=linger-backstop`) (#1982); `allowPrivateHttpProviders` is an opt-in (#2984): when true, plain-HTTP provider baseUrls on private networks (RFC1918, 127/8, link-local, IPv6 ULA/link-local, `*.local`/`.lan`/`.internal`) are admitted by the agent-registry intake and summary-endpoint guards that otherwise trust loopback plaintext only — public IPs/domains and CGNAT (100.64/10) stay refused regardless, and the default `false` keeps pre-#2984 behavior byte-for-byte; `caFile` (#3776) points bili's own upstream fetch at an extra root-CA PEM — merged with, never replacing, the built-in roots and `NODE_EXTRA_CA_CERTS` — for security-scanner/corporate-MITM re-signed upstreams. Full details in the [env table](#config-file-keys-for-environment-knobs-2030). ### `persist` @@ -1707,6 +1708,7 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_CHAIN_CONTENT` | `chainContentDetection` | false | | `BILI_CHAIN_STAMP` | `chainEgressStamp` | false | | `BILI_CLAUDE_NATIVE_PORT` | `claude.nativePort` | unset (lane sticky zone port) | +| `BILI_CA_FILE` | `network.caFile` | unset | | `BILI_CLIENT_ERROR_BACKSTOP_MS` | `network.clientErrorBackstopMs` | 30000 | | `BILI_CODEARTS_REF` | `resign` | {} (armed; built-in scheme sdk-hmac-sha256) | | `BILI_CODEX_COMPACT` | `codexCompact` | "intercept" | @@ -1840,6 +1842,7 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_REPLAY_RETRY_MAX` | Total attempts for replay retries (default `3`; set `1` to disable retries entirely — legacy single-attempt behavior). See #189. On the acp-loop/preflight/compress loops it applies to both transient HTTP failures and #1263 fail-fast network failures (pre-response reset/refused — proxy-recycle class), never to timeout/abort kinds. On the main model-request path (#1688) only fail-fast pre-response network failures are replayed; HTTP verdicts (4xx/5xx) always pass through verbatim, untried again. | | `BILI_PROXY_KEEPALIVE_MAX_MS` | Keep-alive reuse ceiling (ms) for connections through an upstream proxy (default `55000`). Common proxies recycle idle tunnels on a ~60s cadade; capping our reuse window below that trades a few reconnects for the classic "first request after idle dies with ECONNRESET" failure (#1263). `0` = uncapped (undici defaults). Direct (non-proxied) connections are unaffected. | | `BILI_UPSTREAM_TIMEOUT_MS` | Idle budget (ms) for upstream requests: time-to-first-byte and time between body chunks (default `720000` = 12 min). A healthy stream that keeps producing chunks is never cut mid-flight; a silent one is. The same value drives the underlying HTTP client's transport timeouts, so this single knob bounds long local-model prefills end-to-end (#551). | +| `BILI_CA_FILE` | Path to a PEM file with extra root CAs for bili's own upstream TLS trust (#3776). Security software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy re-signs the upstream with its own CA: browsers trust it via the OS certificate store, but Node's fetch reads only its built-in roots plus `NODE_EXTRA_CA_CERTS` — so bili fails with `SELF_SIGNED_CERT_IN_CHAIN` while the rest of the machine connects fine. Export the scanner/vendor root to PEM and set this (or `NODE_EXTRA_CA_CERTS`); alternatively exclude node.exe from the software's HTTPS inspection. The file is MERGED with Node's built-in roots and any `NODE_EXTRA_CA_CERTS` entries — never replaces them. Unset = today's default trust, byte-for-byte. File-config key: `network.caFile`. | | ~~`BILI_STREAM_STALL_MS`~~ | **Removed in #1714** (the #1452 opt-in streaming-phase stall guard is retired). In #1706 a stale `400` export turned every thinking-phase silence into a false truncation; local-model deployments legitimately go silent for minutes mid-stream (thinking phases, long prefills), so no finite sub-budget was safe for everyone. Upstream silence — prefill and mid-stream alike — is now bounded solely by the always-on `BILI_UPSTREAM_TIMEOUT_MS` idle budget (default 12 min, tunable). Old values are ignored; bili names a stale export once at startup (`no longer read`) — remove it from your shell profile. | | `BILI_KEEP_ALIVE_TIMEOUT_MS` | Keep-alive timeout (ms) for client-facing sockets (default `5000`, matching Node's implicit default; #1452). Idle client connections are reaped with a clean FIN by Node's built-in reaper; this makes the previously implicit value explicit and configurable, and the reap is classified as `reason=idle-timeout` in the connection lifecycle ledger (debug log). Non-numeric or non-positive values fall back to `5000`. | | `BILI_EXPOSURE_LOG_INTERVAL_MS` | Interval (ms) for the periodic long-lived-exposure telemetry line `[exposure] uptime=… liveConns=… tcpHandles=… handles=… sessions=… blindTunnels=… inFlight=…` (default `3600000` = hourly; #1452). `0` disables. Exists so socket-handle leaks and zombie connections surface in long-running logs instead of post-mortem forensics. | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index aa15fc2bdd..45495c793a 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -159,6 +159,7 @@ | `network.decodedBodyBytes` | number | 10737418240 | BILI_DECODED_BODY_BYTES | content-encoding 解压(gzip/deflate/br/zstd)后的解码字节上限,解码过程中增量检查(decode 阶段 413)。默认为接收预算的 2 倍,沿用旧的 decode=2×wire 关系;用于防解压炸弹。 | | `network.outboundBodyBytes` | number | 5368709120 | BILI_OUTBOUND_BODY_BYTES | 转发到上游的重建请求字节上限(forward 阶段 413),按最终 body 字符串的 UTF-8 字节计。在 fetch 之前检查,超预算的重建不会触发上游看门狗分配。 | | `network.wsOutboundBytes` | number | 10485760 | BILI_WS_OUTBOUND_BYTES | 仅针对 WebSocket 通道的软字节预算(#3701):当最终序列化的 Responses WS 帧超出该预算时,优先从出站副本中裁减较早的图片(归档为占位符备注——原始文件仍保留在会话存储中,可通过 read_original_image / decompress 恢复),随后对剩余内容通过 jimp maxDim 阶梯进行有界降采样。若帧仍超出限制且上游在任何事件发出前即以 1009 关闭连接,则在同一请求内尝试一次严格更小的重发。设为 0 可禁用裁减(即恢复此前的原样发送行为)。HTTP 通道不受影响——硬性拒绝上限仍为 network.outboundBodyBytes。 | +| `network.caFile` | string | 未设置 | BILI_CA_FILE | PEM 文件路径,包含要额外信任的根 CA(#3776),仅作用于 bili 自己的上游 fetch。与 Node 内置根证书及 `NODE_EXTRA_CA_CERTS` 中的条目合并——从不替换。安全软件做 HTTPS/加密连接扫描(卡巴斯基、企业 DLP)或企业 MITM 代理重签上游时使用:把该厂商根证书导出为 PEM 并指向此键。未设置 = 默认信任,行为逐字节不变。 | | `guardrails` | object | {} (all defaults) | — | 守卫集合:pi/omp 插件的工具输出守卫(#3066)及代理侧流式 text-run 守卫(#3406)。 | | `guardrails.bashTimeout` | number | 60 | BILI_GUARDRAIL_BASH_TIMEOUT | 模型未写 timeout 时注入到 bash 工具的秒数(#3066 owner 有意为之,#3366 重申:无人看管的 bash 会把 pi 会话挂死一整天——宿主 schema 本身无默认值)。设 N 调整秒数,设 0 关闭注入(恢复 pi 原生行为)。 | | `guardrails.maxOutputBytes` | number | 50000 | BILI_GUARDRAIL_MAX_OUTPUT_BYTES | pi/omp 插件中每个工具结果的文本输出合计字节上限(0 禁用)。超限输出从头截断并附指向宿主已存完整输出的提示。 | @@ -883,7 +884,7 @@ - **类型:** `{ upstreamTimeoutMs?: number; requestWatchdogMs?: number; replayRetryMax?: number; replayRetryBaseMs?: number; maxShrinkPerCompress?: number; keepAliveTimeoutMs?: number; clientErrorBackstopMs?: number; exposureLogIntervalMs?: number; streamKeepAliveMs?: number; preflightHoldMs?: number; preflightDeadEndCooldownMs?: number; proxyKeepAliveMaxMs?: number; postResponseLingerMs?: number; allowPrivateHttpProviders?: boolean }` - **默认:** `{}`(内置值依次为:`720000`、`2× 上游超时`、`3`、`1500`、未设置、`5000`、`30000`、`3600000`、`15000`、`30000`、`300000`、`55000`、`5000`、`false`) - **状态:** ACTIVE -- **说明:** 进程级传输与重试时序(毫秒)。每个键按 env > file > default 解析,对应环境变量分别为 `BILI_UPSTREAM_TIMEOUT_MS`、`BILI_REQUEST_WATCHDOG_MS`、`BILI_REPLAY_RETRY_MAX`、`BILI_REPLAY_RETRY_BASE_MS`、`BILI_MAX_SHRINK_PER_COMPRESS`、`BILI_KEEP_ALIVE_TIMEOUT_MS`、`BILI_CLIENT_ERROR_BACKSTOP_MS`、`BILI_EXPOSURE_LOG_INTERVAL_MS`、`BILI_STREAM_KEEPALIVE_MS`、`BILI_PREFLIGHT_HOLD_MS`、`BILI_PREFLIGHT_DEAD_END_COOLDOWN_MS`、`BILI_PROXY_KEEPALIVE_MAX_MS`、`BILI_POST_RESPONSE_LINGER_MS`、`BILI_ALLOW_PRIVATE_HTTP_PROVIDERS`。要点:`upstreamTimeoutMs` 端到端约束首字节时间与 chunk 间静默(#551);`requestWatchdogMs` 是单请求总预算(负值 = 退出该约束);`replayRetryMax` 统计瞬态上游拒绝后的尝试次数(#189/#1688);`maxShrinkPerCompress` 限制单次压缩对请求的最大缩减比例,超限则引导模型选择更小的范围(#189);`streamKeepAliveMs` 在响应零字节写出超过该时长时发一条 SSE 注释行(#1647);`proxyKeepAliveMaxMs` 把经上游代理的连接复用窗口压在上游回收周期之下(#1263),`0` 取消上限;`postResponseLingerMs` 是代理主动发起的响应后关闭的优雅关闭预算 —— socket 被挂起等待对端 FIN/TLS close_notify,到点销毁(`reason=linger-backstop`)(#1982);`allowPrivateHttpProviders` 是可选开关(#2984):为 true 时,私网(RFC1918、127/8、link-local、IPv6 ULA/link-local、`*.local`/`.lan`/`.internal`)上的明文 HTTP provider baseUrl 被 agent-registry 接收与摘要端点守卫放行——这些守卫默认只信任回环明文;公网 IP/域名与 CGNAT(100.64/10)无论如何都拒;默认 false 保持 #2984 之前的行为逐字不变。完整细节见[环境变量表](#环境变量的配置键对照-2030)。 +- **说明:** 进程级传输与重试时序(毫秒)。每个键按 env > file > default 解析,对应环境变量分别为 `BILI_UPSTREAM_TIMEOUT_MS`、`BILI_REQUEST_WATCHDOG_MS`、`BILI_REPLAY_RETRY_MAX`、`BILI_REPLAY_RETRY_BASE_MS`、`BILI_MAX_SHRINK_PER_COMPRESS`、`BILI_KEEP_ALIVE_TIMEOUT_MS`、`BILI_CLIENT_ERROR_BACKSTOP_MS`、`BILI_EXPOSURE_LOG_INTERVAL_MS`、`BILI_STREAM_KEEPALIVE_MS`、`BILI_PREFLIGHT_HOLD_MS`、`BILI_PREFLIGHT_DEAD_END_COOLDOWN_MS`、`BILI_PROXY_KEEPALIVE_MAX_MS`、`BILI_POST_RESPONSE_LINGER_MS`、`BILI_ALLOW_PRIVATE_HTTP_PROVIDERS`、`BILI_CA_FILE`。要点:`upstreamTimeoutMs` 端到端约束首字节时间与 chunk 间静默(#551);`requestWatchdogMs` 是单请求总预算(负值 = 退出该约束);`replayRetryMax` 统计瞬态上游拒绝后的尝试次数(#189/#1688);`maxShrinkPerCompress` 限制单次压缩对请求的最大缩减比例,超限则引导模型选择更小的范围(#189);`streamKeepAliveMs` 在响应零字节写出超过该时长时发一条 SSE 注释行(#1647);`proxyKeepAliveMaxMs` 把经上游代理的连接复用窗口压在上游回收周期之下(#1263),`0` 取消上限;`postResponseLingerMs` 是代理主动发起的响应后关闭的优雅关闭预算 —— socket 被挂起等待对端 FIN/TLS close_notify,到点销毁(`reason=linger-backstop`)(#1982);`allowPrivateHttpProviders` 是可选开关(#2984):为 true 时,私网(RFC1918、127/8、link-local、IPv6 ULA/link-local、`*.local`/`.lan`/`.internal`)上的明文 HTTP provider baseUrl 被 agent-registry 接收与摘要端点守卫放行——这些守卫默认只信任回环明文;公网 IP/域名与 CGNAT(100.64/10)无论如何都拒;默认 false 保持 #2984 之前的行为逐字不变;`caFile`(#3776)让 bili 自己的上游 fetch 额外信任一个根 CA PEM 文件——与内置根证书和 `NODE_EXTRA_CA_CERTS` 合并,从不替换——用于安全扫描软件/企业 MITM 重签的上游。完整细节见[环境变量表](#环境变量的配置键对照-2030)。 ### `persist` @@ -1706,6 +1707,7 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_CHAIN_CONTENT` | `chainContentDetection` | false | | `BILI_CHAIN_STAMP` | `chainEgressStamp` | false | | `BILI_CLAUDE_NATIVE_PORT` | `claude.nativePort` | unset (lane sticky zone port) | +| `BILI_CA_FILE` | `network.caFile` | 未设置 | | `BILI_CLIENT_ERROR_BACKSTOP_MS` | `network.clientErrorBackstopMs` | 30000 | | `BILI_CODEARTS_REF` | `resign` | {} (armed; built-in scheme sdk-hmac-sha256) | | `BILI_CODEX_COMPACT` | `codexCompact` | "intercept" | @@ -1865,6 +1867,7 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_UPSTREAM_PROXY` | 代理自身出站连接的上游代理 —— 在*全局*来源中优先级最高(高于 Web UI 手动代理与配置文件 `proxy`)。per-URL 的 `providers..proxy` 对其匹配的 provider URL 仍然优先。完整解析顺序、防环规则与示例见 [服务端设置 → `proxy`](#服务端设置)。 | | `BILI_INHERITED_HTTP_PROXY` / `BILI_INHERITED_HTTPS_PROXY` / `BILI_INHERITED_ALL_PROXY` / `BILI_INHERITED_NO_PROXY` | 非用户直接使用 —— launcher 起代理子进程时自动设置(#1012)。launcher 会从客户端和代理子进程两侧剥掉 shell 的代理变量(客户端必须把流量发给 bili;代理的模型出网也不能被 shell 代理劫持),但会把用户剥离前的代理转发到这些变量里,让代理的**辅助出网**(MITM 盲隧道 —— 客户端侧的 MCP/web 流量)仍能走用户的 VPN。它们只作用于盲隧道的 fallback 层:显式路由 / 全局 `proxy` / `BILI_UPSTREAM_PROXY` / 显式 `"upstreamProxyMode": "direct"` 仍然优先,指向 bili 自身端口的值会被丢弃。模型出网不受影响(未显式配置则保持直连)。 | | `BILI_UPSTREAM_TIMEOUT_MS` | 上游请求的空闲预算(毫秒):首字节时间(TTFB)与响应体块之间的间隔(默认 `720000` = 12 分钟)。持续产出数据块的健康流永远不会被中途切断;静默的流才会。同一个值同时驱动底层 HTTP 客户端的传输层超时,因此这一个旋钮即可端到端约束本地大模型的超长 prefill(#551)。 | +| `BILI_CA_FILE` | 指向一个 PEM 文件的路径,包含要额外信任的根 CA(#3776),仅作用于 bili 自己的上游 TLS。做 HTTPS/加密连接扫描的安全软件(卡巴斯基、企业 DLP)或企业 MITM 代理会用自家 CA 重签上游:浏览器通过系统证书库信任它,但 Node 的 fetch 只读内置根证书加 `NODE_EXTRA_CA_CERTS`——于是 bili 报 `SELF_SIGNED_CERT_IN_CHAIN`,而机器上其他一切连接正常。把扫描软件/厂商根证书导出为 PEM 后设置此项(或 `NODE_EXTRA_CA_CERTS`);或在安全软件的 HTTPS 检查中排除 node.exe。该文件与 Node 内置根证书及 `NODE_EXTRA_CA_CERTS` 条目合并——从不替换。未设置 = 当前默认信任,逐字节不变。配置键:`network.caFile`。 | | `BILI_ATTACH_HEALTH_DEADLINE_MS` | dsh/opencode attach 校验中,attach 目标已挂但本进程模型通道**钉死**在其上(观察到指向它的 `/bili/…` 路由流量)时的健康等待上限(毫秒):bili 等待目标恢复而不是 spawn 第二实例——spawn 会把会话劈成两半(模型流量保持钉死,bili 工具在另一实例上 404)。超时后大声报错,并在每次模型请求时持续重查直到目标恢复(默认 `15000`)。见 #1365。 | | `BILI_ATTACH_EVIDENCE_GRACE_MS` | dsh/opencode attach 校验探测到目标已挂时,等待路由通道证据出现的宽限窗口(毫秒),超时才回退到旧的 spawn 路径(覆盖「判定早于首个请求」的竞态:t≈0 时探测失败、t≈1s 时首个模型请求才落地)(默认 `5000`)。见 #1365。 | | `BILI_PERSIST` | 设 `0` 关闭会话持久化(仅内存,重启即丢)。 | diff --git a/src/config.ts b/src/config.ts index 59fae46f62..4218ab0fc1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1884,6 +1884,12 @@ type FileConfig = { outboundBodyBytes?: number; /** Soft egress byte budget for the WebSocket lane (#3701; default 10 MiB, 0 disables). Above it, older non-protected images are shed from the outbound copy before sending. Env BILI_WS_OUTBOUND_BYTES wins. */ wsOutboundBytes?: number; + /** Extra root-CA PEM bundle trusted by bili's OWN outbound upstream fetch (#3776): for security + * software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate + * MITM proxy whose re-signed upstream cert Node's default trust rejects with + * SELF_SIGNED_CERT_IN_CHAIN. MERGED with Node's built-in roots, never replacing them. + * Env BILI_CA_FILE wins. */ + caFile?: string; }; /** Session persistence knobs (#2030) — was BILI_PERSIST_* env-only. */ persist?: { diff --git a/src/fetch-util.ts b/src/fetch-util.ts index 505287b2df..08854ec230 100644 --- a/src/fetch-util.ts +++ b/src/fetch-util.ts @@ -1,6 +1,7 @@ import { Agent } from "undici"; import { currentFetchTransport } from "./fetch-transport.js"; import { classifyUpstreamFailure, isFailFastUpstreamKind } from "./upstream-fail.js"; +import { _resetUpstreamCaForTest, upstreamCaIdentity, upstreamExtraCa } from "./upstream-ca.js"; import { upstreamTimeoutMs as knobUpstreamTimeoutMs, replayMaxAttempts as knobReplayMaxAttempts, @@ -76,13 +77,17 @@ export function upstreamTimeoutMs(): number { // killed long prefills before this watchdog ever got a chance (#551). Inject an // explicit Agent per timeout value so the transport layer matches the watchdog // instead of firing first. -const directDispatchers = new Map(); +const directDispatchers = new Map(); function directDispatcher(timeoutMs: number): Agent { - let agent = directDispatchers.get(timeoutMs); + // #3776: key on the CA identity too so a configured/rotated network.caFile + // rotates the cached Agent (and its connect.ca) instead of reusing a stale trust set. + const key = `${timeoutMs}\u0000${upstreamCaIdentity()}`; + let agent = directDispatchers.get(key); if (!agent) { - agent = new Agent({ headersTimeout: timeoutMs, bodyTimeout: timeoutMs }); - directDispatchers.set(timeoutMs, agent); + const ca = upstreamExtraCa(); + agent = new Agent({ headersTimeout: timeoutMs, bodyTimeout: timeoutMs, ...(ca ? { connect: { ca } } : {}) }); + directDispatchers.set(key, agent); } return agent; } @@ -92,6 +97,7 @@ export function _resetFetchUtilForTest(): void { try { void agent.close().catch(() => undefined); } catch { /* already closed */ } } directDispatchers.clear(); + _resetUpstreamCaForTest(); } /** undici's fetch accepts a `dispatcher` option (its own Dispatcher type) that diff --git a/src/knobs.ts b/src/knobs.ts index 52810e8577..393cc39ac6 100644 --- a/src/knobs.ts +++ b/src/knobs.ts @@ -83,6 +83,19 @@ export function upstreamTimeoutMs(): number { return Number.isInteger(v) && v > 0 ? v : UPSTREAM_TIMEOUT_MS; } +/** BILI_CA_FILE > network.caFile > unset. Path to a PEM bundle of extra root CAs + * trusted by bili's OWN outbound upstream fetch — for security software doing + * HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate + * MITM proxy whose re-signed upstream cert Node's default trust (built-ins + + * NODE_EXTRA_CA_CERTS) rejects with SELF_SIGNED_CERT_IN_CHAIN (#3776). The PEM is + * MERGED with Node's built-in roots, never replacing them (#3776). */ +export function upstreamCaFile(): string | undefined { + const envRaw = process.env["BILI_CA_FILE"]; + if (envRaw !== undefined && envRaw !== "") return envRaw; + const fileValue = fileNetwork().caFile; + return typeof fileValue === "string" && fileValue !== "" ? fileValue : undefined; +} + /** BILI_REPLAY_RETRY_MAX > network.replayRetryMax > 3. */ export function replayMaxAttempts(): number { const v = tNum("BILI_REPLAY_RETRY_MAX", fileNetwork().replayRetryMax); diff --git a/src/preflight.ts b/src/preflight.ts index c7f28f47c5..d0397e51d1 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -17,6 +17,7 @@ import { applyRanges, normalizeRangeOrder, type RewriteCtx } from "./stream.js"; import { fetchWithTimeout, isTransientUpstreamError, replayMaxAttempts, replayBackoffMs, sleep, UpstreamHttpError } from "./fetch-util.js"; import { dumpSummaryRejection } from "./error-dump.js"; import { proxyDispatcher } from "./upstream-proxy.js"; +import { isTlsTrustCode, tlsTrustRemediation, TLS_TRUST_CODES } from "./upstream-fail.js"; import { lastCompressSuffix, type Session } from "./session.js"; import { epochConfirmPending } from "./fold-reconcile.js"; import { peekRegistryOutputLimit } from "./registry.js"; @@ -1210,15 +1211,7 @@ const TRANSIENT_SUMMARY_CODES = new Set(["ECONNRESET", "ECONNREFUSED", "EPIPE", const SUMMARY_DIAGNOSTIC_CODES = new Set([ ...TRANSIENT_SUMMARY_CODES, "ETIMEDOUT", "UND_ERR_ABORTED", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT", "Z_DATA_ERROR", "Z_BUF_ERROR", "Z_MEM_ERROR", - "ERR_TLS_CERT_ALTNAME_INVALID", "CERT_HAS_EXPIRED", "DEPTH_ZERO_SELF_SIGNED_CERT", "DEPTH_ZERO_UNTRUSTED_ROOT", - "SELF_SIGNED_CERT_IN_CHAIN", "UNABLE_TO_VERIFY_LEAF_SIGNATURE", "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", -]); -// Certificate-TRUST codes: the failure is "this CA chain is not trusted here" -// (corporate interception / MITM), not "the upstream is broken" — the -// client-facing error must say so and name the remedy (#1987). -const TLS_TRUST_HINT_CODES = new Set([ - "ERR_TLS_CERT_ALTNAME_INVALID", "CERT_HAS_EXPIRED", "DEPTH_ZERO_SELF_SIGNED_CERT", "DEPTH_ZERO_UNTRUSTED_ROOT", - "SELF_SIGNED_CERT_IN_CHAIN", "UNABLE_TO_VERIFY_LEAF_SIGNATURE", "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", + ...TLS_TRUST_CODES, // #3776: single source of truth lives in upstream-fail.js ]); class SummaryTransportError extends Error { @@ -2624,9 +2617,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // the client regardless of which code surfaced (a cert problem is not more // terminal than a gzip corruption). Only an abort (client gone / idle budget) // is not worth retrying; keep that case's field omitted as before. - const caNote = err.code !== undefined && TLS_TRUST_HINT_CODES.has(err.code) - ? " This looks like TLS certificate interception (corporate proxy/MITM): install the corporate root CA for Node.js via NODE_EXTRA_CA_CERTS=/path/to/ca.pem, or run node with --use-system-ca." - : ""; + const caNote = isTlsTrustCode(err.code) ? ` — ${tlsTrustRemediation()}` : ""; failure = { kind: "upstream", detail: `the summarization call failed: ${err.message}${caNote}`, ...(err.aborted ? {} : { retryable: true }) }; deps.log("warn", `[preflight] summarization failed: ${err.message}`); } else { diff --git a/src/server.ts b/src/server.ts index 43e693179d..e9ea9c09eb 100644 --- a/src/server.ts +++ b/src/server.ts @@ -17,7 +17,7 @@ import { contextFromRegistry, loadRegistry, peekRegistryContext, peekRegistryOut import { codexAlignedWindow } from "./codex-models.js"; import { fetchWithTimeout, upstreamTimeoutMs } from "./fetch-util.js"; import { getUpstreamConnectionStatus, resolveProxy, resolveProxyDecision, proxyDispatcher, type UpstreamProxyDecision } from "./upstream-proxy.js"; -import type { SkippedProxyInfo } from "./upstream-fail.js"; +import { upstreamFailureOf, type SkippedProxyInfo } from "./upstream-fail.js"; import { getUpstreamAlerts } from "./upstream-alerts.js"; import { hostIdForLog, maskHostPortForLog, setMaskHostsEnabled, maskUrlForLog, maskUrlsInText } from "./log-mask.js"; // Protocol codecs + the historical-image strip primitive live in the kernel now @@ -643,8 +643,17 @@ export async function startServer(opts: ProxyOptions): Promise { else log("error", msg); if (!res.headersSent) { const status = msg.includes("exceeds") ? 413 : 502; + // #3776 sub-defect 3: surface the machine-readable failure slice + // (kind/code/hint) alongside the free-text detail so clients can + // self-diagnose without reading bili's log. Additive — when no + // upstream failure is attached the body is byte-identical to before. + const uf = upstreamFailureOf(err); + const diag: Record = {}; + if (uf?.kind !== undefined) diag.kind = uf.kind; + if (uf?.code !== undefined) diag.code = uf.code; + if (uf?.hint !== undefined) diag.hint = uf.hint; res.writeHead(status, { "content-type": "application/json" }); - res.end(JSON.stringify({ error: "acp-proxy failure", detail: msg })); + res.end(JSON.stringify({ error: "acp-proxy failure", detail: msg, ...(Object.keys(diag).length > 0 ? { upstreamFailure: diag } : {}) })); } else { res.end(); } diff --git a/src/server/relay.ts b/src/server/relay.ts index eb024f15ad..73ba89356a 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -48,8 +48,9 @@ import { rewriteResponsesJsonResponseAsync } from "../stream-responses.js"; import { observeResponsesTerminalState } from "../stream-terminal.js"; import { rewriteJsonResponseAsync, type RewriteCtx } from "../stream.js"; import { safePrefix, safeSuffix, scrubLoneSurrogatesOnWire } from "../text-safe.js"; -import { clearUpstreamAlertsForHost, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason } from "../upstream-alerts.js"; -import { formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js"; +import { clearUpstreamAlertsForHost, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason, upstreamCircuitLastFailureKind } from "../upstream-alerts.js"; +import { describeUpstreamFailure, formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js"; +import { withUpstreamFailure, type UpstreamFailureKind } from "../upstream-fail.js"; import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js"; import { recordPckAblation } from "../pck-probe.js"; import { applyOutputSteering, applyOutputSteeringJson } from "../output-steering.js"; @@ -676,7 +677,11 @@ export async function forward( log("warn", `[${prepared?.session.id ?? "passthrough"}] [acp-proxy] upstream circuit open — failing fast without contacting ${upstreamUrl}: ${circuitBlock}`); if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "upstream circuit open", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate); if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream circuit open"); - throw new Error(`upstream circuit open: ${circuitBlock}`); + const lastKind = upstreamCircuitLastFailureKind(upstreamUrl); + throw withUpstreamFailure(new Error(`upstream circuit open: ${circuitBlock}`), { + ...(lastKind !== undefined ? { kind: lastKind as UpstreamFailureKind } : {}), + hint: "upstream circuit open — the endpoint is unreachable and bili is failing fast instead of resending full context until a probe succeeds", + }); } let upstreamResult: Awaited>; // #3701: the catch below replaces upstreamResult on a shrunken resend; capture the @@ -704,15 +709,15 @@ export async function forward( if (oversize === undefined) { recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy); recordUpstreamAlert(upstreamUrl, error, proxyUrl !== undefined, skippedProxy); - const failureDetail = formatUpstreamError(error, upstreamUrl, proxyUrl, skippedProxy); - recordUpstreamCircuitFailure(upstreamUrl, error, proxyUrl !== undefined, failureDetail); + const failure = describeUpstreamFailure(error, upstreamUrl, proxyUrl, skippedProxy); + recordUpstreamCircuitFailure(upstreamUrl, error, proxyUrl !== undefined, failure.message); // #604: a network-level failure (socket reset, timeout abort) also never // reports usage — arm the emergency shrink like the 5xx branch below. if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "network failure", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate); // [#1343] no response means the attached full text never reached the model — // drop-and-log it (a corrective note surfaces on the next qualifying request). if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream network failure"); - throw new Error(`upstream request failed: ${failureDetail}`, { cause: error }); + throw withUpstreamFailure(new Error(`upstream request failed: ${failure.message}`, { cause: error }), failure.detail); } // #3701: close 1009 with ZERO events received is a verdict on our frame SIZE, not // on endpoint health — the same gateway answers smaller frames fine. Deliberately @@ -721,6 +726,7 @@ export async function forward( // token-space shrink for visibility, then try ONE bounded byte-shrink resend that // must be strictly smaller than what was just rejected. recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy); + if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "network failure", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate); if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream network failure"); const sentBytes = typeof wireBody === "string" ? Buffer.byteLength(wireBody) : 0; diff --git a/src/upstream-alerts.ts b/src/upstream-alerts.ts index 82053f14f0..405f7fb1bb 100644 --- a/src/upstream-alerts.ts +++ b/src/upstream-alerts.ts @@ -177,6 +177,15 @@ export function upstreamCircuitBlockReason(url: string, now: number = Date.now() return undefined; } +/** #3776 sub-defect 3: the kind of the last failure that opened a host's + * circuit, so the egress can surface WHY requests are failing fast (the same + * kind the user must fix). undefined when no open circuit / no recorded failure. */ +export function upstreamCircuitLastFailureKind(url: string): string | undefined { + const entry = circuits.get(hostOf(url)); + if (!entry || entry.state.openedAt === null) return undefined; + return entry.state.lastFailure?.kind; +} + /** Record one FINAL pre-response transport failure (the error escaped the * in-request retry ladder). Same discipline as ALERT_KINDS: client aborts, * post-connect timeouts and HTTP verdicts never COUNT toward opening — only diff --git a/src/upstream-ca.ts b/src/upstream-ca.ts new file mode 100644 index 0000000000..1277c17d5e --- /dev/null +++ b/src/upstream-ca.ts @@ -0,0 +1,73 @@ +// #3776 sub-defect 2: CA trust material for bili's OWN outbound upstream fetch. +// Node's default TLS trust is the built-in Mozilla bundle + NODE_EXTRA_CA_CERTS; +// it does NOT read the OS certificate store, so a security-software MITM root +// (Kaspersky / corporate DLP) installed in the Windows/Mac store is untrusted and +// every request dies with SELF_SIGNED_CERT_IN_CHAIN (#3773). network.caFile / +// BILI_CA_FILE points at that vendor's PEM; we MERGE it with the built-ins (never +// replace) so all other upstreams keep working. When unset we return undefined and +// the dispatcher uses Node's default trust byte-for-byte (no connect override). + +import { readFileSync, statSync } from "node:fs"; +import * as tls from "node:tls"; +import { log as loggerLog } from "./logger.js"; +import { upstreamCaFile } from "./knobs.js"; + +const IDENTITY_TTL_MS = 30_000; +let lastStatAt = 0; +let lastIdentity = "none"; +let cacheKey: string | null = null; +let cacheValue: string[] | undefined; + +function readPem(path: string): string[] { + try { + return [readFileSync(path, "utf8")]; + } catch (err) { + loggerLog("warn", `[upstream-ca] CA file ${path} unreadable (${(err as Error).message}); ignoring — upstream TLS falls back to whatever else is trusted`); + return []; + } +} + +/** Stable identity of the configured extra-CA material ("none" when unset), + * TTL-cached so a statSync happens at most once per IDENTITY_TTL_MS even on the + * hot fetch path. Used both as the dispatcher cache-key discriminator and as the + * PEM rebuild trigger, so a rotated CA file is picked up without a restart. */ +export function upstreamCaIdentity(now = Date.now()): string { + const caFile = upstreamCaFile(); + if (!caFile) { + lastIdentity = "none"; + return "none"; + } + if (now - lastStatAt < IDENTITY_TTL_MS && lastIdentity !== "none") return lastIdentity; + let mtimeMs = 0; + try { mtimeMs = statSync(caFile).mtimeMs; } catch { /* missing file => mtime 0 */ } + lastIdentity = `${caFile}\u0000${mtimeMs}`; + lastStatAt = now; + return lastIdentity; +} + +/** Merged CA list (built-ins + NODE_EXTRA_CA_CERTS + network.caFile) for bili's + * own fetch, or undefined when no caFile is configured — in which case the + * dispatcher must NOT set connect.ca and Node's default trust applies unchanged. + * The merge (not replace) matters: an explicit undici connect.ca REPLACES the + * default store, so we fold in everything Node would already have trusted. */ +export function upstreamExtraCa(): string[] | undefined { + const identity = upstreamCaIdentity(); + if (identity === "none") return undefined; + if (cacheKey === identity) return cacheValue; + const merged: string[] = [...tls.rootCertificates]; + const nodeExtra = process.env["NODE_EXTRA_CA_CERTS"]; + if (nodeExtra) for (const p of nodeExtra.split(":")) if (p.trim() !== "") merged.push(...readPem(p.trim())); + const caFile = upstreamCaFile(); + if (caFile) merged.push(...readPem(caFile)); + cacheKey = identity; + cacheValue = merged; + loggerLog("info", `[upstream-ca] upstream TLS trust extended with ${caFile} (${merged.length} certs total)`); + return merged; +} + +export function _resetUpstreamCaForTest(): void { + lastStatAt = 0; + lastIdentity = "none"; + cacheKey = null; + cacheValue = undefined; +} diff --git a/src/upstream-fail.ts b/src/upstream-fail.ts index 32c60165d0..65e227fde0 100644 --- a/src/upstream-fail.ts +++ b/src/upstream-fail.ts @@ -86,6 +86,58 @@ export function classifyUpstreamFailure(error: unknown, ctx: UpstreamFailCtx = { return "unknown"; } +// #3776: certificate-TRUST codes — the failure means "this CA chain is not +// trusted HERE", i.e. the upstream was re-signed by an intermediate Node.js +// does not know (security-software HTTPS scanning / corporate MITM), NOT "the +// upstream is broken". Single source of truth shared by preflight's summary- +// call note so the two channels never drift apart (preflight previously kept +// its own copy of this set). +export const TLS_TRUST_CODES = new Set([ + "ERR_TLS_CERT_ALTNAME_INVALID", "CERT_HAS_EXPIRED", "DEPTH_ZERO_SELF_SIGNED_CERT", "DEPTH_ZERO_UNTRUSTED_ROOT", + "SELF_SIGNED_CERT_IN_CHAIN", "UNABLE_TO_VERIFY_LEAF_SIGNATURE", "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", +]); + +export function isTlsTrustCode(code: unknown): boolean { + return typeof code === "string" && TLS_TRUST_CODES.has(code); +} + +/** #3776: the remediation text for a certificate-TRUST failure. Names the common + * producer (security-software HTTPS/encrypted-connection scanning, corporate + * MITM) and both fixes. Consumed verbatim by the log line, the 502 body, the + * circuit/alert banner and the preflight summary note so they stay word-for- + * word identical — the generic `UPSTREAM_FAIL_HINTS.tls` one-liner was what left + * a Kaspersky-scanned user digging through logs for a day (#3773/#3776). */ +export function tlsTrustRemediation(): string { + return "the presented certificate chain is not trusted by this Node.js process — usually security software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy re-signing the upstream. Fix: export that scanner/vendor root CA to a PEM file and point NODE_EXTRA_CA_CERTS=/path/to/ca.pem at it (or set bili config \"network.caFile\"), or exclude node.exe from the software's HTTPS inspection"; +} + +// #3776 sub-defect 3: machine-readable slice of an upstream failure. Attached +// NON-enumerably to the thrown error so the egress 502 body can expose +// kind/code/hint verbatim instead of making the client parse free text — while +// the error still stringifies EXACTLY as before (the existing `detail` field is +// byte-identical; this only ADDS a structured sibling). A Symbol key keeps it +// off any accidental JSON.stringify of the error elsewhere. +export interface UpstreamFailureDetail { + kind?: UpstreamFailureKind; + code?: string; + hint?: string; +} + +const UPSTREAM_FAILURE = Symbol("upstreamFailure"); + +export function withUpstreamFailure(err: Error, detail: UpstreamFailureDetail): Error { + Object.defineProperty(err, UPSTREAM_FAILURE, { value: detail, enumerable: false }); + return err; +} + +export function upstreamFailureOf(err: unknown): UpstreamFailureDetail | undefined { + if (err !== null && typeof err === "object") { + const value = (err as Record)[UPSTREAM_FAILURE]; + if (value !== null && typeof value === "object") return value as UpstreamFailureDetail; + } + return undefined; +} + /** Fail-fast kinds: the attempt died BEFORE any response byte existed, so a * replay cannot double-deliver anything. Per-attempt cost stays bounded — * milliseconds for resets/refusals/DNS, at most one connect timeout for diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index 05eeccad61..34f42da480 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -6,7 +6,8 @@ import type { ProviderRoutes } from "./config.js"; import { log as loggerLog } from "./logger.js"; import { maskHostInText, maskUrlForLog } from "./log-mask.js"; import { upstreamTimeoutMs } from "./fetch-util.js"; -import { classifyUpstreamFailure, hintWithSkippedProxy, redactProxyUrl, type SkippedProxyInfo } from "./upstream-fail.js"; +import { classifyUpstreamFailure, hintWithSkippedProxy, isTlsTrustCode, redactProxyUrl, tlsTrustRemediation, type SkippedProxyInfo, type UpstreamFailureDetail } from "./upstream-fail.js"; +import { upstreamCaIdentity, upstreamExtraCa } from "./upstream-ca.js"; import { proxyKeepAliveMaxMs as knobProxyKeepAliveMaxMs } from "./knobs.js"; type ParsedHttpProxy = { @@ -363,7 +364,8 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number if (!proxyUrl) return undefined; const t = timeoutMs ?? upstreamTimeoutMs(); const ka = proxyKeepAliveMaxMs(); - const key = `${proxyUrl}\u0000${t}\u0000${ka}`; + // #3776: include the CA identity in the key so a configured/rotated network.caFile rotates the cached agent. + const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${upstreamCaIdentity()}`; let agent = dispatcherCache.get(key); if (!agent) { const withTimeouts = (options: object): object => { @@ -381,9 +383,12 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number factory: (origin: URL, options: object) => new Pool(origin, withTimeouts(options) as ConstructorParameters[1]), clientFactory: (origin: URL, options: object) => new Pool(origin, withTimeouts(options) as ConstructorParameters[1]), }; + // #3776: trust the merged CA bundle for end-to-end TLS to the target through the tunnel. + const ca = upstreamExtraCa(); + const withConnect = ca ? { ...base, connect: { ca } } : base; agent = ka > 0 - ? new ProxyAgent({ ...base, keepAliveTimeout: ka, keepAliveMaxTimeout: ka }) - : new ProxyAgent(base); + ? new ProxyAgent({ ...withConnect, keepAliveTimeout: ka, keepAliveMaxTimeout: ka }) + : new ProxyAgent(withConnect); dispatcherCache.set(key, agent); } return agent; @@ -520,7 +525,14 @@ function errorChain(error: unknown): Array> { return chain; } -export function formatUpstreamError(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo): string { +export interface UpstreamFailureParts { + /** Formatted one-line failure string (log line / 502 `detail` / circuit banner). */ + message: string; + /** Machine-readable slice (#3776 sub-defect 3) surfaced in the egress 502 body. */ + detail: UpstreamFailureDetail; +} + +export function describeUpstreamFailure(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo): UpstreamFailureParts { const chain = errorChain(error); const kind = classifyUpstreamFailure(error, { viaProxy: proxyUrl !== undefined }); const fields = ["code", "errno", "syscall", "address", "port"]; @@ -535,16 +547,31 @@ export function formatUpstreamError(error: unknown, url: string, proxyUrl?: stri } catch { /* not a URL — nothing to mask */ } return s; }; + let codeValue: unknown; for (const field of fields) { const value = chain.find((entry) => entry[field] !== undefined)?.[field]; - if (value !== undefined) parts.push(`${field}=${maskHostIn(String(value))}`); + if (value !== undefined) { + if (field === "code" && codeValue === undefined) codeValue = value; + parts.push(`${field}=${maskHostIn(String(value))}`); + } } const messages = chain.map((entry) => maskHostIn(String(entry.message ?? ""))).filter(Boolean); if (messages.length > 0) parts.push(`message=${messages.join(" <- ")}`); parts.push(`url=${maskUrlForLog(url)}`); parts.push(`proxy=${redactProxyUrl(proxyUrl) ?? "direct"}`); - parts.push(`hint=${hintWithSkippedProxy(kind, skipped)}`); - return parts.join(" "); + let hint = hintWithSkippedProxy(kind, skipped); + // #3776: only TRUST-code TLS failures (interception/MITM) get the detailed + // remediation naming security-software HTTPS scanning — a plain protocol + // break (EPROTO) does not. Flows to log line, 502 body, circuit banner. + if (kind === "tls" && chain.some((entry) => isTlsTrustCode(entry.code))) { + hint = `${hint}; ${tlsTrustRemediation()}`; + } + parts.push(`hint=${hint}`); + return { message: parts.join(" "), detail: { kind, ...(codeValue !== undefined ? { code: String(codeValue) } : {}), hint } }; +} + +export function formatUpstreamError(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo): string { + return describeUpstreamFailure(error, url, proxyUrl, skipped).message; } export function recordUpstreamConnection(url: string, proxyUrl: string | undefined, error?: unknown, skipped?: SkippedProxyInfo): void { diff --git a/src/web/i18n.ts b/src/web/i18n.ts index 81e40d1001..c8f87d88c1 100644 --- a/src/web/i18n.ts +++ b/src/web/i18n.ts @@ -172,7 +172,7 @@ export const MESSAGES: Record> = { "alert.hint.proxy_reset": "代理在响应开始前断开连接——检查代理空闲回收/载荷上限(BILI_PROXY_KEEPALIVE_MAX_MS 可缩短复用窗口);可能已按 BILI_REPLAY_RETRY_MAX 做有界透明重放", "alert.hint.upstream_reset": "上游/网络在响应开始前重置连接——检查上游与本地网络;可能已按 BILI_REPLAY_RETRY_MAX 做有界透明重放", "alert.hint.dns": "域名解析失败——DNS 服务器或主机名拼写错误;可能已按 BILI_REPLAY_RETRY_MAX 做有界透明重放", - "alert.hint.tls": "CONNECT 或上游握手时 TLS/证书失败——检查 CA/代理 MITM 配置", + "alert.hint.tls": "CONNECT 或上游握手时 TLS/证书失败——检查 CA/代理 MITM 配置;若提示证书链不受信任/自签,可能是安全软件的 HTTPS 加密连接扫描(如卡巴斯基、企业 DLP)或企业代理重签了上游——将该根证书导出为 PEM,设置 NODE_EXTRA_CA_CERTS=/path/to/ca.pem(或在 bili 配置 network.caFile),或在安全软件中排除 node.exe", "alert.hint.unknown": "未分类的传输层故障——请附完整错误链反馈", "advisory.restartDesc": ":安全版本已安装到磁盘,但当前进程仍在运行受影响版本。原因:", "advisory.restartHint": "。重启 bili 后生效。", @@ -662,7 +662,7 @@ export const MESSAGES: Record> = { "alert.hint.proxy_reset": "proxy dropped the connection before the response — check proxy idle-recycle/payload limits (BILI_PROXY_KEEPALIVE_MAX_MS can shorten our reuse window); a bounded transparent replay may be attempted (BILI_REPLAY_RETRY_MAX)", "alert.hint.upstream_reset": "upstream/network reset before the response — check upstream and local network; a bounded transparent replay may be attempted (BILI_REPLAY_RETRY_MAX)", "alert.hint.dns": "name resolution failed — DNS server or hostname typo; a bounded transparent replay may be attempted (BILI_REPLAY_RETRY_MAX)", - "alert.hint.tls": "TLS/certificate failure at CONNECT or upstream handshake — CA/proxy MITM config", + "alert.hint.tls": "TLS/certificate failure at CONNECT or upstream handshake — CA/proxy MITM config; if the chain is reported untrusted/self-signed, security software doing HTTPS scanning (Kaspersky, enterprise DLP) or a corporate proxy may be re-signing the upstream — export that root CA to PEM and set NODE_EXTRA_CA_CERTS=/path/to/ca.pem (or bili config network.caFile), or exclude node.exe from scanning", "alert.hint.unknown": "unclassified transport failure — report with full error chain", "advisory.restartDesc": ": the safe version is installed on disk, but this process is still running the affected version. Reason:", "advisory.restartHint": ". Restart bili to activate it.", diff --git a/tests/issue3776-tls-mitm-diagnostics.test.ts b/tests/issue3776-tls-mitm-diagnostics.test.ts new file mode 100644 index 0000000000..2b91f5872a --- /dev/null +++ b/tests/issue3776-tls-mitm-diagnostics.test.ts @@ -0,0 +1,245 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import https from "node:https"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { once } from "node:events"; +import * as tls from "node:tls"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.js"; +import type { ProxyOptions } from "../src/config.js"; +import { SessionStore, _setStoreForTest } from "../src/persist.js"; +import { _setForTest as setRegistryForTest } from "../src/registry.js"; +import { setLogCapture } from "../src/logger.js"; +import { describeUpstreamFailure, formatUpstreamError } from "../src/upstream-proxy.js"; +import { + TLS_TRUST_CODES, + isTlsTrustCode, + tlsTrustRemediation, + withUpstreamFailure, + upstreamFailureOf, +} from "../src/upstream-fail.js"; +import { upstreamCaIdentity, upstreamExtraCa, _resetUpstreamCaForTest } from "../src/upstream-ca.js"; +import { _resetFetchUtilForTest } from "../src/fetch-util.js"; +import { ensureRootCA, rootCaPath, mintHostCert, _resetForTest as _resetCaForTest } from "../src/ca.js"; + +// #3776: security software doing HTTPS scanning (Kaspersky / corporate DLP) MITMs the +// upstream; Node's default trust (built-ins + NODE_EXTRA_CA_CERTS) ignores the OS cert +// store, so bili's own fetch dies with SELF_SIGNED_CERT_IN_CHAIN and the client only ever +// saw a generic 502. These tests pin the three fixes: (1) trust-code TLS failures carry a +// concrete remediation hint on every channel, (2) network.caFile/BILI_CA_FILE merges an +// extra root into bili's own fetch trust, (3) the 502 body carries a machine-readable +// upstreamFailure{kind,code,hint} so users can self-diagnose without reading bili.log. + +const tmpRoot = process.env.TMPDIR && fs.existsSync(process.env.TMPDIR) ? process.env.TMPDIR : os.tmpdir(); +function mkTmp(prefix: string): string { return fs.mkdtempSync(path.join(tmpRoot, prefix)); } + +function netError(code: string, message: string, cause?: Error): Error { + const err = new Error(message) as NodeJS.ErrnoException & { cause?: unknown }; + err.code = code; + if (cause) err.cause = cause; + return err; +} + +// --- sub-defect 1: remediation hint appears ONLY for trust-code TLS failures --- + +test("#3776 sub-defect 1: trust-code TLS failure carries the security-software remediation", () => { + const wrapped = new TypeError("fetch failed", { cause: netError("SELF_SIGNED_CERT_IN_CHAIN", "self-signed certificate in certificate chain") }); + const parts = describeUpstreamFailure(wrapped, "https://api.example.test/v1/messages"); + assert.equal(parts.detail.kind, "tls"); + assert.equal(parts.detail.code, "SELF_SIGNED_CERT_IN_CHAIN"); + assert.match(parts.message, /security software/); + assert.match(parts.message, /NODE_EXTRA_CA_CERTS/); + // the thin format wrapper stays byte-compatible and carries the same hint + assert.match(formatUpstreamError(wrapped, "https://api.example.test/v1/messages"), /security software/); +}); + +test("#3776 sub-defect 1: non-trust TLS failure (EPROTO) does NOT carry the remediation", () => { + const wrapped = new TypeError("fetch failed", { cause: netError("EPROTO", "protocol error") }); + const parts = describeUpstreamFailure(wrapped, "https://api.example.test/v1/messages"); + assert.equal(parts.detail.kind, "tls"); + assert.equal(parts.detail.code, "EPROTO"); + assert.doesNotMatch(parts.message, /security software/); +}); + +test("#3776 sub-defect 1: isTlsTrustCode / TLS_TRUST_CODES are the single source of truth", () => { + assert.ok(isTlsTrustCode("SELF_SIGNED_CERT_IN_CHAIN")); + assert.ok(isTlsTrustCode("UNABLE_TO_GET_ISSUER_CERT_LOCALLY")); + assert.ok(!isTlsTrustCode("EPROTO")); + assert.ok(!isTlsTrustCode(undefined)); + assert.ok(!isTlsTrustCode(42)); + for (const c of TLS_TRUST_CODES) assert.ok(isTlsTrustCode(c)); + assert.match(tlsTrustRemediation(), /Kaspersky/); +}); + +// --- sub-defect 3: structured failure attachment is retrievable and non-enumerable --- + +test("#3776 sub-defect 3: withUpstreamFailure attaches a non-enumerable, retrievable slice", () => { + const detail = { kind: "tls" as const, code: "SELF_SIGNED_CERT_IN_CHAIN", hint: "the hint" }; + const err = withUpstreamFailure(new Error("boom"), detail); + assert.deepEqual(upstreamFailureOf(err), detail); + assert.deepEqual(Object.keys(err), []); // symbol property is non-enumerable + assert.equal(String(err), "Error: boom"); // message form untouched + assert.equal(upstreamFailureOf(new Error("plain")), undefined); + assert.equal(upstreamFailureOf("not-an-error"), undefined); + assert.equal(upstreamFailureOf(null), undefined); +}); + +// --- sub-defect 2: CA material resolution (merge, never replace; unset = default trust) --- + +test("#3776 sub-defect 2: unset caFile => undefined (Node default trust unchanged)", () => { + const prevEnv = process.env["BILI_CA_FILE"]; + delete process.env["BILI_CA_FILE"]; + try { + _resetUpstreamCaForTest(); + assert.equal(upstreamCaIdentity(), "none"); + assert.equal(upstreamExtraCa(), undefined); + } finally { + if (prevEnv === undefined) delete process.env["BILI_CA_FILE"]; else process.env["BILI_CA_FILE"] = prevEnv; + _resetUpstreamCaForTest(); + } +}); + +test("#3776 sub-defect 2: configured caFile merges built-ins + user PEM (never replaces)", () => { + const dir = mkTmp("bili-cafile-test-"); + const pem = "-----BEGIN CERTIFICATE-----\nMIIBfakeuserca\n-----END CERTIFICATE-----\n"; + const caFile = path.join(dir, "extra-ca.pem"); + fs.writeFileSync(caFile, pem); + const prevEnv = process.env["BILI_CA_FILE"]; + process.env["BILI_CA_FILE"] = caFile; + try { + _resetUpstreamCaForTest(); + assert.notEqual(upstreamCaIdentity(), "none"); + const merged = upstreamExtraCa(); + assert.ok(Array.isArray(merged)); + assert.ok(merged!.length >= tls.rootCertificates.length + 1, "built-ins plus at least the user PEM"); + assert.ok(merged!.includes(pem), "user PEM present verbatim"); + assert.ok(merged!.includes(tls.rootCertificates[0]), "built-in roots preserved (merge, not replace)"); + } finally { + if (prevEnv === undefined) delete process.env["BILI_CA_FILE"]; else process.env["BILI_CA_FILE"] = prevEnv; + _resetUpstreamCaForTest(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// --- e2e: a self-signed MITM upstream, through a real in-process bili proxy --- + +function close(server: http.Server | https.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +async function startTlsScenario(withCaFile: boolean): Promise<{ resp: Response; body: string }> { + // isolated XDG_DATA_HOME so the test root CA is generated fresh, then a leaf cert + // for "localhost" signed by it — the shape of a security-scanner re-signed upstream. + const xdg = mkTmp("bili-tls-mitm-xdg-"); + const prevXdg = process.env.XDG_DATA_HOME; + let leafCert = "", leafKey = "", rootPem = ""; + process.env.XDG_DATA_HOME = xdg; + try { + _resetCaForTest(); + ensureRootCA(); + rootPem = fs.readFileSync(rootCaPath(), "utf8"); + const leaf = mintHostCert("localhost"); + leafCert = leaf.certPem; + leafKey = leaf.keyPem; + } finally { + if (prevXdg === undefined) delete process.env.XDG_DATA_HOME; else process.env.XDG_DATA_HOME = prevXdg; + _resetCaForTest(); + } + + const upstreamHandler = (req: http.IncomingMessage, res: http.ServerResponse) => { + req.resume(); + req.on("end", () => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "r1", object: "chat.completion", created: 1, model: "gpt-test", + choices: [{ index: 0, message: { role: "assistant", content: "hi" }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }, + })); + }); + }; + const upstream = https.createServer({ key: leafKey, cert: leafCert }, upstreamHandler); + await new Promise((r) => upstream.listen(0, "127.0.0.1", r)); + const upstreamPort = (upstream.address() as { port: number }).port; + const origin = `https://localhost:${upstreamPort}`; + + const captured: { level: string; msg: string }[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const prevRetryMax = process.env["BILI_REPLAY_RETRY_MAX"]; + process.env["BILI_REPLAY_RETRY_MAX"] = "1"; // single attempt: keep the scenario fast + const prevCaFile = process.env["BILI_CA_FILE"]; + if (withCaFile) { + const caDir = mkTmp("bili-tls-mitm-cafile-"); + const caFile = path.join(caDir, "scanner-root.pem"); + fs.writeFileSync(caFile, rootPem); + process.env["BILI_CA_FILE"] = caFile; + } else { + delete process.env["BILI_CA_FILE"]; + } + _resetFetchUtilForTest(); // drop any cached dispatcher so the new CA identity takes effect + + const opts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "https://localhost", + routes: { [origin]: { models: { "gpt-test": { context: 400_000 } } } }, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + const resp = await fetch(`http://127.0.0.1:${proxyPort}/bili/${origin}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "tls-mitm-1" }, + body: JSON.stringify({ model: "gpt-test", messages: [{ role: "user", content: "hi" }] }), + }); + const body = await resp.text(); + return { resp, body }; + } finally { + setLogCapture(null); + if (prevRetryMax === undefined) delete process.env["BILI_REPLAY_RETRY_MAX"]; else process.env["BILI_REPLAY_RETRY_MAX"] = prevRetryMax; + if (prevCaFile === undefined) delete process.env["BILI_CA_FILE"]; else process.env["BILI_CA_FILE"] = prevCaFile; + _resetFetchUtilForTest(); + await close(proxy); + await close(upstream); + } +} + +test("#3776 e2e (A): untrusted MITM upstream => 502 carrying structured upstreamFailure{kind:tls}", async () => { + const { resp, body } = await startTlsScenario(false); + assert.equal(resp.status, 502); + const parsed = JSON.parse(body) as { error?: string; detail?: string; upstreamFailure?: { kind?: string; code?: string; hint?: string } }; + assert.equal(parsed.error, "acp-proxy failure"); + assert.ok(parsed.detail?.includes("kind=tls"), `detail should name the kind, got: ${body}`); + assert.equal(parsed.upstreamFailure?.kind, "tls"); + assert.ok(parsed.upstreamFailure?.code, "structured code present"); + assert.match(parsed.upstreamFailure?.hint ?? "", /security software/); +}); + +test("#3776 e2e (B): same MITM upstream trusted via network.caFile => 200 completion", async () => { + const { resp, body } = await startTlsScenario(true); + assert.equal(resp.status, 200); + const parsed = JSON.parse(body) as { choices?: { message?: { content?: string } }[] }; + assert.equal(parsed.choices?.[0]?.message?.content, "hi"); +}); diff --git a/tests/preflight-transport-recovery.test.ts b/tests/preflight-transport-recovery.test.ts index 01a877af9c..9a30fa5adf 100644 --- a/tests/preflight-transport-recovery.test.ts +++ b/tests/preflight-transport-recovery.test.ts @@ -246,7 +246,7 @@ test("preflight marks TLS trust failures client-retryable without in-loop retry assert.equal(result.failure?.retryable, true, `${code}: client-retryable`); if (withHint) { assert.match(result.failure?.detail ?? "", /NODE_EXTRA_CA_CERTS/); - assert.match(result.failure?.detail ?? "", /--use-system-ca/); + assert.match(result.failure?.detail ?? "", /security software/); } else { assert.doesNotMatch(result.failure?.detail ?? "", /NODE_EXTRA_CA_CERTS/); } From fa86bdd2eb9deded610479eeeb681b5f2e4fd4e7 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 14:27:09 +0800 Subject: [PATCH 2/5] docs: seed network.caFile in config-reference yaml (CI docs-sync/docs-coverage/config-surface-guard) (#3776) Review fix for PR #3786: the new knob was documented by hand-editing the GENERATED artifacts (CONFIGURATION.md / zh-CN) while the single source of truth website/config-reference/*.yaml stayed untouched, which trips all three config-doc CI gates (docs-sync strips the hand-added rows on regeneration; docs-coverage flags BILI_CA_FILE undocumented; config-surface-guard flags network.caFile without a seed entry). Add the seed entry and regenerate; the committed table rows now match the generator output byte-for-byte. --- CONFIGURATION.md | 4 ++-- CONFIGURATION.zh-CN.md | 4 ++-- website/config-reference/process.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index da998ca57c..afa68d16cd 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -159,7 +159,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `network.decodedBodyBytes` | number | 10737418240 | BILI_DECODED_BODY_BYTES | Max DECODED bytes after content-encoding expansion (gzip/deflate/br/zstd), checked incrementally during decode (decode-stage 413). Default is 2x the receive budget, mirroring the legacy decode=2x-wire relationship; guards against decompression bombs. | | `network.outboundBodyBytes` | number | 5368709120 | BILI_OUTBOUND_BODY_BYTES | Max rebuilt request bytes forwarded upstream (forward-stage 413), counted as UTF-8 bytes of the final body string. Checked before the fetch so an over-budget rebuild never reaches the upstream watchdog allocation. | | `network.wsOutboundBytes` | number | 10485760 | BILI_WS_OUTBOUND_BYTES | Soft byte budget for the WebSocket lane only (#3701): when the final serialized Responses WS frame exceeds it, older images are shed from the OUTBOUND COPY first (archived as placeholder notes — originals stay in session storage, recoverable via read_original_image / decompress), then whatever remains is boundedly downscaled through the jimp maxDim ladder. If the frame still does not fit and the upstream closes with 1009 before any event, one strictly smaller resend is attempted within the same request. Set 0 to disable shedding (previous send-as-is behavior). HTTP lanes are unaffected — the hard refusal cap stays network.outboundBodyBytes. | -| `network.caFile` | string | unset | BILI_CA_FILE | Path to a PEM file with extra root CAs trusted by bili's OWN upstream fetches (#3776). Merged with Node's built-in roots and any `NODE_EXTRA_CA_CERTS` entries — never replaces them. For security software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy that re-signs the upstream: export that root CA to PEM and point this at it. Unset = default trust, byte-identical behavior. | +| `network.caFile` | string | unset | BILI_CA_FILE | Path to a PEM file with extra root CAs trusted by bili's OWN upstream fetches (#3776). Merged with Node's built-in roots and any NODE_EXTRA_CA_CERTS entries — never replaces them. For security software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy that re-signs the upstream: export that root CA to PEM and point this at it. Unset = default trust, byte-identical behavior. | | `guardrails` | object | {} (all defaults) | — | Guardrails: tool-output guardrails for the pi/omp plugin bundles (#3066) plus the proxy-side text-run stream guard (#3406). | | `guardrails.bashTimeout` | number | 60 | BILI_GUARDRAIL_BASH_TIMEOUT | Seconds injected as the bash tool's timeout when the model omits it (#3066 owner intent, re-affirmed #3366: an unsupervised bash call could hang a pi session for a whole day — the host schema provides no default). Set N to tune, 0 to disable injection (pi-native no-default). | | `guardrails.maxOutputBytes` | number | 50000 | BILI_GUARDRAIL_MAX_OUTPUT_BYTES | Combined text-output byte cap per tool result in the pi/omp plugin (0 disables the cap). Over-cap output is truncated head-first with a notice pointing at the host-saved full output. | @@ -1704,11 +1704,11 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_ALLOW_DSH_COMPACTION` | `dsh.allowDshCompaction` | false | | `BILI_ALLOW_PRIVATE_HTTP_PROVIDERS` | `network.allowPrivateHttpProviders` | false | | `BILI_AUTO_MODE_FORGE` | `autoModeForge.enabled` | true | +| `BILI_CA_FILE` | `network.caFile` | unset | | `BILI_CCR_RETRIEVAL_TTL_MS` | `ccrRetrievalTtlMs` | 600000 (0 disables retrieval) | | `BILI_CHAIN_CONTENT` | `chainContentDetection` | false | | `BILI_CHAIN_STAMP` | `chainEgressStamp` | false | | `BILI_CLAUDE_NATIVE_PORT` | `claude.nativePort` | unset (lane sticky zone port) | -| `BILI_CA_FILE` | `network.caFile` | unset | | `BILI_CLIENT_ERROR_BACKSTOP_MS` | `network.clientErrorBackstopMs` | 30000 | | `BILI_CODEARTS_REF` | `resign` | {} (armed; built-in scheme sdk-hmac-sha256) | | `BILI_CODEX_COMPACT` | `codexCompact` | "intercept" | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 45495c793a..2afa8eb81b 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -159,7 +159,7 @@ | `network.decodedBodyBytes` | number | 10737418240 | BILI_DECODED_BODY_BYTES | content-encoding 解压(gzip/deflate/br/zstd)后的解码字节上限,解码过程中增量检查(decode 阶段 413)。默认为接收预算的 2 倍,沿用旧的 decode=2×wire 关系;用于防解压炸弹。 | | `network.outboundBodyBytes` | number | 5368709120 | BILI_OUTBOUND_BODY_BYTES | 转发到上游的重建请求字节上限(forward 阶段 413),按最终 body 字符串的 UTF-8 字节计。在 fetch 之前检查,超预算的重建不会触发上游看门狗分配。 | | `network.wsOutboundBytes` | number | 10485760 | BILI_WS_OUTBOUND_BYTES | 仅针对 WebSocket 通道的软字节预算(#3701):当最终序列化的 Responses WS 帧超出该预算时,优先从出站副本中裁减较早的图片(归档为占位符备注——原始文件仍保留在会话存储中,可通过 read_original_image / decompress 恢复),随后对剩余内容通过 jimp maxDim 阶梯进行有界降采样。若帧仍超出限制且上游在任何事件发出前即以 1009 关闭连接,则在同一请求内尝试一次严格更小的重发。设为 0 可禁用裁减(即恢复此前的原样发送行为)。HTTP 通道不受影响——硬性拒绝上限仍为 network.outboundBodyBytes。 | -| `network.caFile` | string | 未设置 | BILI_CA_FILE | PEM 文件路径,包含要额外信任的根 CA(#3776),仅作用于 bili 自己的上游 fetch。与 Node 内置根证书及 `NODE_EXTRA_CA_CERTS` 中的条目合并——从不替换。安全软件做 HTTPS/加密连接扫描(卡巴斯基、企业 DLP)或企业 MITM 代理重签上游时使用:把该厂商根证书导出为 PEM 并指向此键。未设置 = 默认信任,行为逐字节不变。 | +| `network.caFile` | string | unset | BILI_CA_FILE | PEM 文件路径,包含要额外信任的根 CA(#3776),仅作用于 bili 自己的上游 fetch。与 Node 内置根证书及 NODE_EXTRA_CA_CERTS 中的条目合并——从不替换。安全软件做 HTTPS/加密连接扫描(卡巴斯基、企业 DLP)或企业 MITM 代理重签上游时使用:把该厂商根证书导出为 PEM 并指向此键。未设置 = 默认信任,行为逐字节不变。 | | `guardrails` | object | {} (all defaults) | — | 守卫集合:pi/omp 插件的工具输出守卫(#3066)及代理侧流式 text-run 守卫(#3406)。 | | `guardrails.bashTimeout` | number | 60 | BILI_GUARDRAIL_BASH_TIMEOUT | 模型未写 timeout 时注入到 bash 工具的秒数(#3066 owner 有意为之,#3366 重申:无人看管的 bash 会把 pi 会话挂死一整天——宿主 schema 本身无默认值)。设 N 调整秒数,设 0 关闭注入(恢复 pi 原生行为)。 | | `guardrails.maxOutputBytes` | number | 50000 | BILI_GUARDRAIL_MAX_OUTPUT_BYTES | pi/omp 插件中每个工具结果的文本输出合计字节上限(0 禁用)。超限输出从头截断并附指向宿主已存完整输出的提示。 | @@ -1703,11 +1703,11 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_ALLOW_DSH_COMPACTION` | `dsh.allowDshCompaction` | false | | `BILI_ALLOW_PRIVATE_HTTP_PROVIDERS` | `network.allowPrivateHttpProviders` | false | | `BILI_AUTO_MODE_FORGE` | `autoModeForge.enabled` | true | +| `BILI_CA_FILE` | `network.caFile` | unset | | `BILI_CCR_RETRIEVAL_TTL_MS` | `ccrRetrievalTtlMs` | 600000 (0 disables retrieval) | | `BILI_CHAIN_CONTENT` | `chainContentDetection` | false | | `BILI_CHAIN_STAMP` | `chainEgressStamp` | false | | `BILI_CLAUDE_NATIVE_PORT` | `claude.nativePort` | unset (lane sticky zone port) | -| `BILI_CA_FILE` | `network.caFile` | 未设置 | | `BILI_CLIENT_ERROR_BACKSTOP_MS` | `network.clientErrorBackstopMs` | 30000 | | `BILI_CODEARTS_REF` | `resign` | {} (armed; built-in scheme sdk-hmac-sha256) | | `BILI_CODEX_COMPACT` | `codexCompact` | "intercept" | diff --git a/website/config-reference/process.yaml b/website/config-reference/process.yaml index 10087d3343..ba0c47e6c2 100644 --- a/website/config-reference/process.yaml +++ b/website/config-reference/process.yaml @@ -138,6 +138,13 @@ entries: group: process desc_en: "Soft byte budget for the WebSocket lane only (#3701): when the final serialized Responses WS frame exceeds it, older images are shed from the OUTBOUND COPY first (archived as placeholder notes — originals stay in session storage, recoverable via read_original_image / decompress), then whatever remains is boundedly downscaled through the jimp maxDim ladder. If the frame still does not fit and the upstream closes with 1009 before any event, one strictly smaller resend is attempted within the same request. Set 0 to disable shedding (previous send-as-is behavior). HTTP lanes are unaffected — the hard refusal cap stays network.outboundBodyBytes." desc_zh: "仅针对 WebSocket 通道的软字节预算(#3701):当最终序列化的 Responses WS 帧超出该预算时,优先从出站副本中裁减较早的图片(归档为占位符备注——原始文件仍保留在会话存储中,可通过 read_original_image / decompress 恢复),随后对剩余内容通过 jimp maxDim 阶梯进行有界降采样。若帧仍超出限制且上游在任何事件发出前即以 1009 关闭连接,则在同一请求内尝试一次严格更小的重发。设为 0 可禁用裁减(即恢复此前的原样发送行为)。HTTP 通道不受影响——硬性拒绝上限仍为 network.outboundBodyBytes。" + - key: network.caFile + type: string + default: "unset" + env: BILI_CA_FILE + group: process + desc_en: "Path to a PEM file with extra root CAs trusted by bili's OWN upstream fetches (#3776). Merged with Node's built-in roots and any NODE_EXTRA_CA_CERTS entries — never replaces them. For security software doing HTTPS/encrypted-connection scanning (Kaspersky, enterprise DLP) or a corporate MITM proxy that re-signs the upstream: export that root CA to PEM and point this at it. Unset = default trust, byte-identical behavior." + desc_zh: "PEM 文件路径,包含要额外信任的根 CA(#3776),仅作用于 bili 自己的上游 fetch。与 Node 内置根证书及 NODE_EXTRA_CA_CERTS 中的条目合并——从不替换。安全软件做 HTTPS/加密连接扫描(卡巴斯基、企业 DLP)或企业 MITM 代理重签上游时使用:把该厂商根证书导出为 PEM 并指向此键。未设置 = 默认信任,行为逐字节不变。" - key: guardrails type: object default: "{} (all defaults)" From 93288dc6a7bc1bffb97ba32623727f73e9c561a8 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 14:32:15 +0800 Subject: [PATCH 3/5] fix: honor Windows NODE_EXTRA_CA_CERTS separators in the CA merge; pin merge + circuit-kind coverage (#3776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for PR #3786: - upstream-ca.ts: the merged trust list re-parses NODE_EXTRA_CA_CERTS itself (an explicit undici connect.ca replaces the default store, so every source Node would already have trusted must be folded in). It split on ':' only, which on Windows shreds drive-letter paths (C:\certs\ca.pem -> "C" + "\certs\ca.pem") and silently drops those certs with a warn log. Mirror Node's own win32 separator handling (';' also separates there). - tests: pin the previously uncovered corners of sub-defects 2 and 3 — NODE_EXTRA_CA_CERTS entries land in the merged list, and upstreamCircuitLastFailureKind reports the opener's kind only while the circuit is open (the kind source of the circuit-open egress hint). --- src/upstream-ca.ts | 6 ++- tests/issue3776-tls-mitm-diagnostics.test.ts | 41 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/upstream-ca.ts b/src/upstream-ca.ts index 1277c17d5e..c46d672b61 100644 --- a/src/upstream-ca.ts +++ b/src/upstream-ca.ts @@ -56,7 +56,11 @@ export function upstreamExtraCa(): string[] | undefined { if (cacheKey === identity) return cacheValue; const merged: string[] = [...tls.rootCertificates]; const nodeExtra = process.env["NODE_EXTRA_CA_CERTS"]; - if (nodeExtra) for (const p of nodeExtra.split(":")) if (p.trim() !== "") merged.push(...readPem(p.trim())); + // Node itself also accepts ';' as a list separator on Windows; splitting on + // ':' alone would shred drive-letter paths (C:\certs\ca.pem -> "C" + "\certs...") + // and silently drop those certs from the merge. + const extraSep = process.platform === "win32" ? /[;:]/ : /:/; + if (nodeExtra) for (const p of nodeExtra.split(extraSep)) if (p.trim() !== "") merged.push(...readPem(p.trim())); const caFile = upstreamCaFile(); if (caFile) merged.push(...readPem(caFile)); cacheKey = identity; diff --git a/tests/issue3776-tls-mitm-diagnostics.test.ts b/tests/issue3776-tls-mitm-diagnostics.test.ts index 2b91f5872a..8084158728 100644 --- a/tests/issue3776-tls-mitm-diagnostics.test.ts +++ b/tests/issue3776-tls-mitm-diagnostics.test.ts @@ -22,6 +22,7 @@ import { upstreamFailureOf, } from "../src/upstream-fail.js"; import { upstreamCaIdentity, upstreamExtraCa, _resetUpstreamCaForTest } from "../src/upstream-ca.js"; +import { recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitLastFailureKind, _resetUpstreamCircuitsForTest } from "../src/upstream-alerts.js"; import { _resetFetchUtilForTest } from "../src/fetch-util.js"; import { ensureRootCA, rootCaPath, mintHostCert, _resetForTest as _resetCaForTest } from "../src/ca.js"; @@ -87,6 +88,20 @@ test("#3776 sub-defect 3: withUpstreamFailure attaches a non-enumerable, retriev assert.equal(upstreamFailureOf(null), undefined); }); +test("#3776 sub-defect 3: upstreamCircuitLastFailureKind exposes the opener's kind only while open", () => { + _resetUpstreamCircuitsForTest(); + const url = "https://circuit-kind.invalid/v1/messages"; + assert.equal(upstreamCircuitLastFailureKind(url), undefined, "fresh host: no open circuit"); + const tlsErr = new TypeError("fetch failed", { cause: netError("SELF_SIGNED_CERT_IN_CHAIN", "self-signed certificate in certificate chain") }); + recordUpstreamCircuitFailure(url, tlsErr, false); + recordUpstreamCircuitFailure(url, tlsErr, false); + assert.equal(upstreamCircuitLastFailureKind(url), undefined, "2 consecutive failures: not open yet"); + recordUpstreamCircuitFailure(url, tlsErr, false); + assert.equal(upstreamCircuitLastFailureKind(url), "tls", "open circuit names the failing kind for the egress hint"); + recordUpstreamCircuitSuccess(url); + assert.equal(upstreamCircuitLastFailureKind(url), undefined, "closed circuit: no kind"); +}); + // --- sub-defect 2: CA material resolution (merge, never replace; unset = default trust) --- test("#3776 sub-defect 2: unset caFile => undefined (Node default trust unchanged)", () => { @@ -124,6 +139,32 @@ test("#3776 sub-defect 2: configured caFile merges built-ins + user PEM (never r } }); +test("#3776 sub-defect 2: NODE_EXTRA_CA_CERTS entries are folded into the merge too", () => { + const dir = mkTmp("bili-cafile-extra-"); + const caPem = "-----BEGIN CERTIFICATE-----\nMIIBcafile\n-----END CERTIFICATE-----\n"; + const caFile = path.join(dir, "ca-file.pem"); + fs.writeFileSync(caFile, caPem); + const extraPem = "-----BEGIN CERTIFICATE-----\nMIIBenvextra\n-----END CERTIFICATE-----\n"; + const extraPath = path.join(dir, "env-extra.pem"); + fs.writeFileSync(extraPath, extraPem); + const prevCa = process.env["BILI_CA_FILE"]; + const prevExtra = process.env["NODE_EXTRA_CA_CERTS"]; + process.env["BILI_CA_FILE"] = caFile; + process.env["NODE_EXTRA_CA_CERTS"] = extraPath; + try { + _resetUpstreamCaForTest(); + const merged = upstreamExtraCa(); + assert.ok(merged!.includes(caPem), "network.caFile PEM present"); + assert.ok(merged!.includes(extraPem), "NODE_EXTRA_CA_CERTS PEM present (merge, not replace)"); + assert.ok(merged!.includes(tls.rootCertificates[0]), "built-ins preserved"); + } finally { + if (prevCa === undefined) delete process.env["BILI_CA_FILE"]; else process.env["BILI_CA_FILE"] = prevCa; + if (prevExtra === undefined) delete process.env["NODE_EXTRA_CA_CERTS"]; else process.env["NODE_EXTRA_CA_CERTS"] = prevExtra; + _resetUpstreamCaForTest(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + // --- e2e: a self-signed MITM upstream, through a real in-process bili proxy --- function close(server: http.Server | https.Server): Promise { From db1e13589c894e37b7e51ef1d717a54ac3912289 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 14:40:16 +0800 Subject: [PATCH 4/5] fix: unexport UpstreamFailureParts (CI unused-exports gate) (#3776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix for PR #3786: the interface is only consumed inside src/upstream-proxy.ts as the return type of describeUpstreamFailure — no other module imports it, so the unused-exports gate fails the PR head. Drop the leading export (INTERNAL_ONLY); callers keep the shape via structural inference, no behavior or typing change. --- src/upstream-proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index 34f42da480..c1f1ab7660 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -525,7 +525,7 @@ function errorChain(error: unknown): Array> { return chain; } -export interface UpstreamFailureParts { +interface UpstreamFailureParts { /** Formatted one-line failure string (log line / 502 `detail` / circuit banner). */ message: string; /** Machine-readable slice (#3776 sub-defect 3) surfaced in the egress 502 body. */ From a28f0a23ace11a66f5b0206e43a29fd305e9b7c7 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 15:45:14 +0800 Subject: [PATCH 5/5] fix: mirror Node's single-file NODE_EXTRA_CA_CERTS semantics in the CA merge (#3776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix for PR #3786 (replaces the separator heuristic from 93288dc6a, which this commit supersedes): Node does NOT split NODE_EXTRA_CA_CERTS on any separator — verified against Node 22, the entire value is loaded as ONE file (multi-cert files carry several PEM blocks). The previous ':' / win32 ';:' splits diverged from Node on every platform and shredded drive-letter paths on Windows (which also broke the env-extra test there). Read the value as a single path so the merge trusts exactly what Node would already have trusted; pin the no-split semantics with an extra assertion. --- src/upstream-ca.ts | 10 +++++----- tests/issue3776-tls-mitm-diagnostics.test.ts | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/upstream-ca.ts b/src/upstream-ca.ts index c46d672b61..40f5fd3623 100644 --- a/src/upstream-ca.ts +++ b/src/upstream-ca.ts @@ -56,11 +56,11 @@ export function upstreamExtraCa(): string[] | undefined { if (cacheKey === identity) return cacheValue; const merged: string[] = [...tls.rootCertificates]; const nodeExtra = process.env["NODE_EXTRA_CA_CERTS"]; - // Node itself also accepts ';' as a list separator on Windows; splitting on - // ':' alone would shred drive-letter paths (C:\certs\ca.pem -> "C" + "\certs...") - // and silently drop those certs from the merge. - const extraSep = process.platform === "win32" ? /[;:]/ : /:/; - if (nodeExtra) for (const p of nodeExtra.split(extraSep)) if (p.trim() !== "") merged.push(...readPem(p.trim())); + // Node treats NODE_EXTRA_CA_CERTS as ONE file path — no list separator on any + // platform (verified against Node 22: the whole value is loaded as a single + // file, multi-cert files carry several PEM blocks). Mirror that exactly so + // the merge trusts precisely what Node would already have trusted. + if (nodeExtra) merged.push(...readPem(nodeExtra)); const caFile = upstreamCaFile(); if (caFile) merged.push(...readPem(caFile)); cacheKey = identity; diff --git a/tests/issue3776-tls-mitm-diagnostics.test.ts b/tests/issue3776-tls-mitm-diagnostics.test.ts index 8084158728..62879318cd 100644 --- a/tests/issue3776-tls-mitm-diagnostics.test.ts +++ b/tests/issue3776-tls-mitm-diagnostics.test.ts @@ -157,6 +157,13 @@ test("#3776 sub-defect 2: NODE_EXTRA_CA_CERTS entries are folded into the merge assert.ok(merged!.includes(caPem), "network.caFile PEM present"); assert.ok(merged!.includes(extraPem), "NODE_EXTRA_CA_CERTS PEM present (merge, not replace)"); assert.ok(merged!.includes(tls.rootCertificates[0]), "built-ins preserved"); + // Node loads NODE_EXTRA_CA_CERTS as ONE file path (no list separator on any + // platform) — a colon-joined value is one nonexistent file to Node itself, + // so the merge must NOT split it and fold in the first half. + _resetUpstreamCaForTest(); + process.env["NODE_EXTRA_CA_CERTS"] = `${extraPath}:${path.join(dir, "bogus.pem")}`; + const joined = upstreamExtraCa(); + assert.ok(!joined!.includes(extraPem), "colon-joined env value treated as a single path (mirrors Node)"); } finally { if (prevCa === undefined) delete process.env["BILI_CA_FILE"]; else process.env["BILI_CA_FILE"] = prevCa; if (prevExtra === undefined) delete process.env["NODE_EXTRA_CA_CERTS"]; else process.env["NODE_EXTRA_CA_CERTS"] = prevExtra;