From bc50f1a87162f2589e16a84a6631e9f6b68ffec4 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 24 Oct 2026 20:30:43 +0800 Subject: [PATCH 01/13] =?UTF-8?q?fix(#4516):=20give=20the=20degraded=20dir?= =?UTF-8?q?ect=20fallback=20its=20missing=20guards=20=E2=80=94=20oversize?= =?UTF-8?q?=20refusal,=20self-heal=20retry,=20transition=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateless native lanes (pi/omp/opencode-V1) had none of the safeguards their siblings carry when the proxy goes away: a provably-oversized raw history rode straight to the provider (1.06M tokens vs a 500K window in the field report), a failed bootstrap meant direct-for-process-lifetime, and the only trace was a single warn line. All three contracts now live in the shared intercept so every lane gets them: - size arm on directSendOrRefuse: the doom math is hoisted out of dsh into the intercept (media-aware two-stage pricing #4111-direction, #4079 floored bar, fail-open on missing data); provably-oversized bodies get a local 422 (x-bili-gate: oversize-refusal-fallback, retryable, wire-shaped per dialect) instead of a guaranteed upstream rejection. - self-heal: a request finding no live origin fires the owner's singleFlight respawn under a 15s cooldown and then holds out for the in-flight attempt under state.spawnWaitMs ?? readyTimeoutMs ?? 15s — the pi spawn lane sets SPAWN_BUDGET_MS+10s so a slow cold start (#4515) is waited out instead of refused at the generic cap. Recovery needs no host restart. - announce: every proxy/direct/refused TRANSITION logs exactly one line (the old one-shot warned flag is gone); state.lastUnavailableReason carries bootstrap-failed vs dead into the refusal wording. dsh keeps its gate-facing signature (a wrapper decodes init.body); the hoisted functions are re-exported so existing importers and tests are untouched. TECHNICAL-NOTES.md gains the degraded-fallback contract section. Behavior changes vs past releases: 1. Gateless lanes now REFUSE locally (422) what provably cannot fit the window while degraded — previously such bodies were sent direct and died at the provider. Bodies that fit, or where the window/body is unknown, still go direct exactly as before. 2. A failed bootstrap is retried per request under a 15s cooldown instead of staying down until the host process restarts. 3. The pi spawn lane waits SPAWN_BUDGET_MS+10s (70s) for readiness on the retry path instead of the generic 15s cap. --- TECHNICAL-NOTES.md | 37 +++ src/agent/dsh-native.ts | 39 ++- src/agent/native-intercept.ts | 249 +++++++++++++++--- src/agent/pi-native.ts | 9 +- tests/issue4516-pi-direct-fallback.test.ts | 279 +++++++++++++++++++++ tests/native-intercept.test.ts | 20 +- 6 files changed, 574 insertions(+), 59 deletions(-) create mode 100644 tests/issue4516-pi-direct-fallback.test.ts diff --git a/TECHNICAL-NOTES.md b/TECHNICAL-NOTES.md index 10b2ef94e8..202d9c5aa5 100644 --- a/TECHNICAL-NOTES.md +++ b/TECHNICAL-NOTES.md @@ -384,6 +384,43 @@ remove ` best-effort queries the live proxy (instance file → `GET /__bili/sessions`, 2.5 s timeout, silent on failure) and prints a note when sessions of that client were active within the last **10 minutes**. +## Degraded direct fallback — refuse, self-heal, announce (#4516, #3304, #2924, #4079) + +When a native lane's proxy is unavailable, model traffic falls back to direct +sends (`directSendOrRefuse`, `src/agent/native-intercept.ts`). That path is a +degraded mode with three contracts, all shared by every lane (the gateless +lanes pi/omp/opencode-V1 had none of them before #4516 — a 1.06M-token raw +history rode straight to the provider): + +- **Size arm** — before ANY degraded send, the body is priced with the same + doom math as dsh's gate (`bodyProvablyOverWindow`): stage 1 is the raw + byte floor vs the window, stage 2 strips structural media bytes (#4111 + direction: media-heavy bodies whose core TEXT fits are NOT refused). The + bar never assumes a window below `MIN_ASSUMED_WINDOW_TOKENS` (1M, #4079 — + declared windows can be wrong-small); bodies in that band are left to + preflight. Missing data (no `x-bili-plugin-context-window` header, + unreadable body) fails OPEN to the direct send — a local guard must never + false-block on its own ignorance. Verdict: 422 + `x-bili-gate: oversize-refusal-fallback`, retryable, wire-shaped per + upstream dialect, message names floor/core/window and the unavailability + reason (`bootstrap-failed` vs `dead` — the latter only after the lane held + an origin and lost it). +- **Self-heal** — a failed bootstrap no longer means + direct-for-process-lifetime. Each request that finds no live origin fires + the owner's `respawn` (singleFlight — concurrent requests share ONE + attempt) under a 15 s cooldown, then holds out for the in-flight attempt + under `state.spawnWaitMs ?? readyTimeoutMs ?? 15_000`: the pi spawn lane + sets `SPAWN_BUDGET_MS + 10 s` so a slow cold start (#4515) is waited out + instead of refused at the generic cap. Recovery needs no host restart. +- **Announce** — every TRANSITION into/out of the degraded state logs one + line to bili.log (proxy recovered / goes direct / refused), not one line + per request; `state.lastUnavailableReason` carries WHY into the refusal + wording so the user can tell "still starting" from "died". + +The opencode V2 resolver lanes (`createLiveOriginResolver`) keep their own +probe-driven retry; this contract covers the plain intercept path they do +not. Attribution reporting into the panel remains dsh-only (see #4516). + ## Unified ACP tag-echo remediation — the four invariants (#2023/#2066/#2190/#2248) One PR folds the three remediation legs together (refs-run residue #2023, diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index ec1a6f0f6c..e10f001bc2 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -59,7 +59,7 @@ import { adminRelayEnabled, resolveResignSettings } from "../config.js"; import { dshContextTabEnabled, dshContextTabLabel } from "../knobs.js"; import { MIN_ASSUMED_WINDOW_TOKENS } from "../util.js"; import { markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, singleFlight } from "./native-bootstrap.js"; -import { installNativeFetchIntercept, liveStateSlotEffect, noteRoutedOrigin, observeRoutedOrigin, routedBiliModelUrl, type GateVerdict, type NativeInterceptState } from "./native-intercept.js"; +import { bodyProvablyOverWindow as sharedBodyProvablyOverWindow, installNativeFetchIntercept, liveStateSlotEffect, noteRoutedOrigin, observeRoutedOrigin, routedBiliModelUrl, type GateVerdict, type NativeInterceptState } from "./native-intercept.js"; import { asciiHeaderValue, fetchManifest, fetchProxyVersion, fetchStatus, fetchStatusLatest, forwardTool, reportAttributionHealth, reportRuntimeInfo, waitForProxyVersion, type ManifestTool } from "./shared.js"; import { createForkAdopter, sideShapedBody } from "./fork-adopt.js"; import { clearIncrementalProbe, createPrepareDirectSend, INCREMENTAL_TURN_MARKER_HEADER, INCREMENTAL_TURN_ID_HEADER, newIncrementalTurnId, noteIncrementalRejection, noteIncrementalAck, planIncrementalTurn, probeIncrementalTurns, rememberOriginalBody, sweepStale, type IncrementalWireMessage } from "./incremental-client.js"; @@ -580,33 +580,24 @@ function effectiveContextWindow(ctx: PluginContext): number | undefined { return modelInfo.byKey.get(modelInfoKey(target.provider, target.model))?.contextWindow; } -/** #4111: the shared two-stage doom predicate for the gate's oversize arms - * (the `threw` arm and the unattributed arm). Stage 1 prices the RAW body at - * bytes/4 — exactly the old math, cheap — and disarms when that is not even - * provisionally over the #4079-floored window. Stage 2 runs ONLY on the - * would-refuse path: re-price on media-excluded core bytes and disarm when - * the core text alone still fits (the #4111 false refusal was media mass - * billed as text). Returns the refusal's numbers when the body is PROVABLY - * over (core text alone cannot fit), undefined otherwise. No body, unreadable - * body, or unresolvable window → undefined — the arms keep #1117 direct - * semantics and can never false-refuse from missing data. */ +// Gate face of the shared doom math (#4111/#4516): keeps the gate's +// init-body signature for its call sites and tests, decoding init.body into +// the raw string the shared form (native-intercept.ts) prices with the +// media-aware walk (media-bytes.ts). +function rawBodyStringOf(init: RequestInit | undefined): string | undefined { + const body: unknown = init?.body; + if (typeof body === "string") return body; + if (body instanceof Uint8Array) return new TextDecoder().decode(body); + if (body instanceof ArrayBuffer) return new TextDecoder().decode(new Uint8Array(body)); + return undefined; +} + export function bodyProvablyOverWindow( body: { bytes: number } | undefined, init: RequestInit | undefined, declaredWindow: number | undefined, -): { coreBytes: number; mediaBytes: number; floorTokens: number; window: number; declaredWindow: number } | undefined { - if (body === undefined || declaredWindow === undefined) return undefined; - const window = Math.max(declaredWindow, MIN_ASSUMED_WINDOW_TOKENS); - if (Math.ceil(body.bytes / 4) <= window) return undefined; - const rawBody: unknown = init?.body; - let raw: string | undefined; - if (typeof rawBody === "string") raw = rawBody; - else if (rawBody instanceof Uint8Array) raw = new TextDecoder().decode(rawBody); - else if (rawBody instanceof ArrayBuffer) raw = new TextDecoder().decode(new Uint8Array(rawBody)); - const mediaBytes = raw === undefined ? 0 : mediaBytesOfWireBody(raw); - const coreBytes = Math.max(body.bytes - mediaBytes, 0); - if (Math.ceil(coreBytes / 4) <= window) return undefined; - return { coreBytes, mediaBytes, floorTokens: Math.ceil(coreBytes / 4), window, declaredWindow }; +): ReturnType { + return sharedBodyProvablyOverWindow(body?.bytes, rawBodyStringOf(init), declaredWindow); } function refreshModelInfo(origin: string | undefined, target?: { provider: string; model: string }): void { diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 3f22eaf97a..09aaec0460 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -10,8 +10,9 @@ import { installFetchChain, isDeadClosureError, resetFetchChainForTest } from ". import { logHostEvent } from "./host-log.js"; import { envMillis } from "./native-bootstrap.js"; +import { mediaBytesOfWireBody } from "./media-bytes.js"; import { probeLiveness, type LivenessVerdict } from "../liveness.js"; -import { BILI_PASSTHROUGH_HEADER } from "../util.js"; +import { BILI_PASSTHROUGH_HEADER, MIN_ASSUMED_WINDOW_TOKENS } from "../util.js"; import { DSH_COMPACTION_MARKER_SNIFF } from "../server/dsh-compaction-guard.js"; import { APIG_RESIGN_HEADER, APIG_RESIGN_CREDENTIAL_HEADER, APIG_RESIGN_SCHEME, clearSignedRefusal, encodeApigCredential, inboundSignedScheme, recordSignedRefusal, resignEnabled, resignPassthroughEnabled, signedRefusal } from "../apig-resign.js"; @@ -59,6 +60,12 @@ export interface NativeInterceptState { * live-but-saturated proxy out of "proxy unavailable" wording; the dsh * give-up seam appends it to the client event line (#4125). */ lastUnavailableReason?: UnavailableReason; + /** #4516: how long a request may hold while an IN-FLIGHT respawn settles + * before degrading to direct/refuse — the owner knows its own bring-up + * budget (pi spawn mode: SPAWN_BUDGET_MS + margin). Without it the + * generic 15s readiness cap would refuse or direct-send a turn that + * would have routed fine had it waited out a slow cold start (#4515). */ + spawnWaitMs?: number; /** Attach mode (#809): route through a user-supplied external proxy at * state.origin instead of a spawned one. Set by the host entry when a * BILLION_CONTEXT_ATTACH / launcher-preset BILLION_CONTEXT_PROXY is @@ -501,6 +508,111 @@ function unattributedOversizeRefusal(url: string): Response { return new Response(JSON.stringify({ error: { type: "server_error", code: "bili_gate_oversize_refusal_unattributed", message, retryable: false } }), { status: 422, headers }); } +// ———— Oversized doom math (#2924/#4068, media-aware since #4111) —————— +// Serves both the dsh takeoverGate arms and the shared direct-fallback arm +// below — the gateless lanes (pi/omp/opencode-V1) have no size judgment at +// all on their degraded path without it (#4516). + +// The media-aware byte walk lives in media-bytes.ts (#4111) — the shared +// home dsh/opencode-native also import from; do not redefine it here. + +/** #4111: the shared two-stage doom predicate for every oversize-refusal arm. + * Stage 1 prices the RAW body at bytes/4 — exactly the old math, cheap — and + * disarms when that is not even provisionally over the #4079-floored window. + * Stage 2 runs ONLY on the would-refuse path: re-price on media-excluded + * core bytes and disarm when the core text alone still fits (the #4111 + * false refusal was media mass billed as text). Returns the refusal's + * numbers when the body is PROVABLY over (core text alone cannot fit), + * undefined otherwise. No body, unreadable body, or unresolvable window → + * undefined — callers keep their direct semantics and can never + * false-refuse from missing data. */ +export function bodyProvablyOverWindow( + bodyBytes: number | undefined, + rawBody: string | undefined, + declaredWindow: number | undefined, +): { coreBytes: number; mediaBytes: number; floorTokens: number; window: number; declaredWindow: number } | undefined { + if (bodyBytes === undefined || declaredWindow === undefined) return undefined; + const window = Math.max(declaredWindow, MIN_ASSUMED_WINDOW_TOKENS); + if (Math.ceil(bodyBytes / 4) <= window) return undefined; + const mediaBytes = rawBody === undefined ? 0 : mediaBytesOfWireBody(rawBody); + const coreBytes = Math.max(bodyBytes - mediaBytes, 0); + if (Math.ceil(coreBytes / 4) <= window) return undefined; + return { coreBytes, mediaBytes, floorTokens: Math.ceil(coreBytes / 4), window, declaredWindow }; +} + +/** The model context window every plugin lane stamps onto its model requests + * (pi/opencode/dsh/omp all do). The shared direct-fallback size arm needs it + * client-side because the proxy — which knows windows server-side — is + * precisely what is unavailable (#4516). */ +const CONTEXT_WINDOW_HEADER = "x-bili-plugin-context-window"; + +function declaredWindowFromHeaders(input: string | URL | Request, init: RequestInit | undefined): number | undefined { + const headers = outgoingHeadersOf(input, init); + if (headers === undefined) return undefined; + const value = headers[CONTEXT_WINDOW_HEADER]; + if (value === undefined) return undefined; + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : undefined; +} + +/** #4516: measure what a direct send would actually put on the wire — byte + * count plus the raw string when decodable (stage-2 media re-pricing needs + * it). Request-object inputs are CLONED before reading (the caller's body + * must stay undisturbed — #2496 discipline). Unreadable bodies (streams, + * FormData) → undefined → the size arm stays disarmed; never a false + * refusal from missing data. */ +async function directSendBodyMeasure(input: string | URL | Request, init: RequestInit | undefined): Promise<{ bytes: number; raw?: string } | undefined> { + const size = bodyByteSizeOf(init); + if (size !== undefined) { + const body: unknown = init?.body; + let raw: string | undefined; + if (typeof body === "string") raw = body; + else if (body instanceof Uint8Array) raw = new TextDecoder().decode(body); + else if (body instanceof ArrayBuffer) raw = new TextDecoder().decode(new Uint8Array(body)); + return { bytes: size.bytes, raw }; + } + if (input instanceof Request) { + try { + const buf = await input.clone().arrayBuffer(); + return { bytes: buf.byteLength, raw: new TextDecoder().decode(buf) }; + } catch { + return undefined; + } + } + return undefined; +} + +/** #4516: local refusal for a DEGRADED direct send the lane proved cannot + * fit its own window. Same 422 policy-refusal class as oversizeRefusal / + * unattributedOversizeRefusal, distinct remedy wording: bili retries the + * proxy automatically (cooldown-gated respawn, see installNativeFetchIntercept), + * so a later retry can genuinely succeed — hence retryable:true, unlike the + * gate arms whose remedy is a host restart. */ +function fallbackOversizeRefusal( + url: string, + doom: { coreBytes: number; mediaBytes: number; floorTokens: number; window: number; declaredWindow: number }, + reason?: UnavailableReason, +): Response { + const why = + reason === "dead" + ? "the bili proxy died and no replacement came up" + : reason === "busy-exhausted" + ? "the bili proxy is alive but unresponsive within the wait budget (#3654)" + : "the bili proxy did not come up (spawn failed or is still starting past the wait budget)"; + const windowNote = + doom.declaredWindow === doom.window + ? `the model context window (${doom.window.toLocaleString("en-US")})` + : `the model context window (${doom.declaredWindow.toLocaleString("en-US")} declared; the local bar never assumes below ${doom.window.toLocaleString("en-US")}, #4079)`; + const message = + `billion-context refused this request locally: ${why}, so this body would go to the provider uncompressed — but its core text alone (~${doom.floorTokens.toLocaleString("en-US")} tok floor estimate from ${doom.coreBytes.toLocaleString("en-US")} B / 4) exceeds ${windowNote}; the provider rejects it and every retry re-sends the same oversized history, so the turn cannot succeed while the proxy stays down (#4516). ` + + "bili keeps retrying the proxy automatically — retry shortly; if this conversation has outgrown the window, start a fresh session."; + const headers = { "content-type": "application/json", "x-bili-gate": "oversize-refusal-fallback" }; + if (url.endsWith("/messages")) { + return new Response(JSON.stringify({ type: "error", error: { type: "invalid_request_error", message } }), { status: 422, headers }); + } + return new Response(JSON.stringify({ error: { type: "server_error", code: "bili_gate_oversize_refusal_fallback", message, retryable: true } }), { status: 422, headers }); +} + async function withTimeout(p: Promise, ms: number): Promise { let timer: ReturnType | undefined; const timeout = new Promise((resolve) => { @@ -622,6 +734,8 @@ const BUSY_REPROBE_BASE_MS = 1_000; * detection horizon is consistent with it while removing the per-request RTT. */ const HEALTH_PROBE_TTL_MS = 2_000; const RESPAWN_COOLDOWN_MS = 15_000; +// #4516: test-only override for the shared intercept's respawn retry cadence. +let respawnCooldownOverrideMs: number | undefined; /** Default liveness probe: GET /__bili/health under the single-shot budget — * now three-state (#3654): TCP connect refused classifies dead in @@ -912,7 +1026,28 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): } // Shared across every re-armed chain link (a re-arm replaces the chain // top but must not forget what this process already learned). - let warned = false; + // #4516: dispatch-mode transition tracking. The old one-shot `warned` + // fired exactly once per process, so every LATER entry into (and exit + // out of) the degraded state was invisible in bili.log. Track the coarse + // mode and log ONLY transitions (bounded noise): entries carry the + // reason, recovery announces itself. By-design direct traffic + // (unattributed passthrough, signed opt-in) never flips the tracker. + let dispatchMode: "proxy" | "direct" | "refused" | undefined; + const noteDispatchMode = (next: "proxy" | "direct" | "refused", detail?: string): void => { + if (dispatchMode === next) return; + const prev = dispatchMode; + dispatchMode = next; + if (next === "proxy") { + if (prev !== undefined) logHostEvent("info", "[native-client]", `bili-native: proxy recovered — model requests resume compression (was ${prev})`); + } else { + logHostEvent("error", "[native-client]", detail ?? `bili-native: model requests now go ${next}`); + } + }; + // #4516: timestamp of the last cooldown-gated respawn retry (main branch) + // and whether this process ever held a live origin (refusal wording: + // "never came up" vs "came up, then died"). + let lastRespawnAttempt = 0; + let everHeldOrigin = false; // Origins verified dead-and-replaced during a runtime recovery (#1130). // Launcher settings overlays bake the origin into request URLs, so after // a replace we must reroute those pre-baked URLs before they touch the @@ -986,13 +1121,17 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): // would send without the guard (usually `input`; makeTarget(routedTarget) // on the routed give-up). Restored bodies ride INIT so fetch-spec // precedence (init over Request) stays exactly what the original - // dispatch had. - const directSendOrRefuse = async (dispatchUrl: string, defaultInput: string | URL | Request, init: RequestInit | undefined): Promise => { + // dispatch had. `degradeLog` marks the DEGRADED call sites (bootstrap + // failed / proxy died / respawn failed): only those flip the dispatch- + // mode tracker, because the other callers are by-design direct traffic, + // not degradation (#4516). + const directSendOrRefuse = async ( + dispatchUrl: string, + defaultInput: string | URL | Request, + init: RequestInit | undefined, + degradeLog?: string, + ): Promise => { const prep = await prepareForDirectSend(dispatchUrl, defaultInput, init); - if (prep.outcome === "as-is") { - state.onDispatch?.(dispatchUrl, "direct"); - return send(defaultInput, init); - } if (prep.outcome === "unrestorable") { // A lane-only payload (e.g. an incremental delta) cannot be // forwarded to a provider that cannot parse it — refusing @@ -1000,12 +1139,37 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): // again (re-probe / re-plan) instead of burning the turn on // an upstream 400. state.onDispatch?.(dispatchUrl, "refused"); + if (degradeLog !== undefined) noteDispatchMode("refused", `bili-native: ${degradeLog} — lane-only payload refused locally instead (#3304)`); logHostEvent("error", "[native-client]", `bili-native: no usable proxy and the request carries a lane-only payload — failing locally instead of sending it to the provider; retry the turn (#3304): ${dispatchUrl}`); throw new TypeError("billion-context: proxy unavailable — the request payload is lane-specific (incremental turn) and cannot be forwarded to the provider directly; retry the turn (#3304)"); } + // #4516: without a live proxy the raw history rides straight to + // the provider — refuse LOCALLY what provably cannot fit the + // model's own window (declared via x-bili-plugin-context-window, + // stamped by every plugin lane): the upstream rejects it and every + // retry re-sends the same oversized history, so the turn can never + // succeed while the proxy stays down. No header / unreadable body / + // body that fits once media is excluded → fail open, exactly like + // the gate arms. + const measured = prep.outcome === "restored" ? { bytes: Buffer.byteLength(prep.body), raw: prep.body } : await directSendBodyMeasure(defaultInput, init); + const doom = measured !== undefined ? bodyProvablyOverWindow(measured.bytes, measured.raw, declaredWindowFromHeaders(defaultInput, init)) : undefined; + if (doom !== undefined) { + state.onDispatch?.(dispatchUrl, "refused"); + if (degradeLog !== undefined) noteDispatchMode("refused", `bili-native: ${degradeLog} — oversized request refused locally instead (~${doom.floorTokens.toLocaleString("en-US")} tok core-text floor > ${doom.window.toLocaleString("en-US")} tok window)`); + logHostEvent("error", "[native-client]", `bili-native: oversized request refused locally instead of direct send — core text ~${doom.floorTokens.toLocaleString("en-US")} tok floor (${doom.coreBytes.toLocaleString("en-US")} B / 4; media ${doom.mediaBytes.toLocaleString("en-US")} B excluded, #4111) exceeds the ${doom.window.toLocaleString("en-US")} tok window: ${dispatchUrl}`); + return fallbackOversizeRefusal(dispatchUrl, doom, state.lastUnavailableReason); + } + if (prep.outcome === "as-is") { + state.onDispatch?.(dispatchUrl, "direct"); + const resp = await send(defaultInput, init); + if (degradeLog !== undefined) noteDispatchMode("direct", `bili-native: ${degradeLog}: ${dispatchUrl}`); + return resp; + } const restored = withRestoredBody(defaultInput, init, prep.body); state.onDispatch?.(dispatchUrl, "direct"); - return send(restored.input, restored.init); + const resp = await send(restored.input, restored.init); + if (degradeLog !== undefined) noteDispatchMode("direct", `bili-native: ${degradeLog}: ${dispatchUrl}`); + return resp; }; // Already-routed `/bili/` model requests (launcher settings overlay): @@ -1070,6 +1234,7 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): } catch { // observability hook only — never break the response } + noteDispatchMode("proxy"); return resp; } catch (err) { // The overlay bakes a specific proxy origin into these URLs; @@ -1104,21 +1269,19 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): const retried = `${again}${u.pathname}${u.search}`; const restamped = withHeaders(makeTarget(retried), init, routedExtra); state.onDispatch?.(retried, "retry"); - const resp = await send(restamped.input, restamped.init); + const resp = await send(restamped.input, stamped.init); try { - state.onResponse?.(url, resp, outgoingHeadersOf(restamped.input, restamped.init)); + state.onResponse?.(url, resp, outgoingHeadersOf(restamped.input, stamped.init)); } catch { // observability hook only — never break the response } + noteDispatchMode("proxy"); return resp; } state.lastUnavailableReason = "send-failed"; state.onGiveUp?.(); - if (!warned) { - warned = true; - logHostEvent("error", "[native-client]", `bili-native: no live proxy — model requests go direct (uncompressed): ${routedTarget}`); - } - return directSendOrRefuse(routedTarget, makeTarget(routedTarget), init); + state.lastUnavailableReason = "dead"; + return directSendOrRefuse(routedTarget, makeTarget(routedTarget), init, "no live proxy — model requests go direct (uncompressed)"); } } if (!isModelApiUrl(url)) { @@ -1225,16 +1388,35 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): } } - const origin = await readyOrigin(state); + let origin = await readyOrigin(state); + if (origin === undefined && state.respawn !== undefined && Date.now() - lastRespawnAttempt >= (respawnCooldownOverrideMs ?? RESPAWN_COOLDOWN_MS)) { + // #4516: after a load-time bootstrap failure NOTHING else retries + // — recover() needs a send failure against a HELD origin, and the + // resolver-based lanes are opencode-only — so a failed spawn used + // to mean direct-for-process-lifetime. Retry the owner's bootstrap + // request-driven, bounded by the same cooldown createLiveOriginResolver + // uses; the owner's respawn is singleFlight, so concurrent requests + // share one attempt instead of stacking spawns. + lastRespawnAttempt = Date.now(); + const attempt = state.respawn(); + state.ready = attempt; + // Hold out for the IN-FLIGHT attempt under the owner's own bring-up + // budget instead of the generic 15s readiness cap: a slow cold + // start (#4515) must not refuse or direct-send a turn that would + // have routed through the proxy had it waited a few seconds longer. + // The owner's singleFlight makes concurrent requests share this one + // attempt — no stacked spawns. + origin = await withTimeout(attempt, state.spawnWaitMs ?? (state.readyTimeoutMs ?? 15_000)); + } if (origin === undefined) { - // Bootstrap failed or timed out — NEVER break the agent: send - // direct (uncompressed) and say so once. - if (!warned) { - warned = true; - logHostEvent("error", "[native-client]", `bili-native: proxy not ready — model request goes direct (uncompressed): ${url}`); - } - return directSendOrRefuse(url, input, init); + // Bootstrap failed or timed out — NEVER break the agent: refuse + // what provably cannot fit, send the rest direct (uncompressed), + // and log every transition into this state (#4516). + state.lastUnavailableReason = everHeldOrigin ? "dead" : "bootstrap-failed"; + return directSendOrRefuse(url, input, init, everHeldOrigin ? "proxy respawn failed — model requests go direct (uncompressed)" : "proxy not ready — model requests go direct (uncompressed)"); } + state.lastUnavailableReason = undefined; + everHeldOrigin = true; // Attach mode rewrites exactly like spawn mode (#809 semantics — // opencode's attach probe+rewrite; the V1 fetch patch relies on it to // catch providers without an explicit baseURL). For dsh under the @@ -1244,7 +1426,9 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): // MITM envs entirely (an http loopback target is never proxied). if (url.startsWith(`${origin}/`)) { state.onDispatch?.(url, "self"); - return send(input, init); + const resp = await send(input, init); + noteDispatchMode("proxy"); + return resp; } await waitToolsGate(); if (state.beforeSend !== undefined) { @@ -1264,6 +1448,7 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): } catch { // observability hook only — never break the response } + noteDispatchMode("proxy"); return resp; } catch (err) { // The proxy can die mid-session (its parent watchdog fires when @@ -1285,6 +1470,7 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): } catch { // observability hook only — never break the response } + noteDispatchMode("proxy"); return resp; } // No replacement available — this session runs direct for its @@ -1293,11 +1479,8 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): // state. state.lastUnavailableReason = "send-failed"; state.onGiveUp?.(); - if (!warned) { - warned = true; - logHostEvent("error", "[native-client]", `bili-native: proxy respawn failed — model requests go direct (uncompressed): ${url}`); - } - return directSendOrRefuse(url, input, init); + state.lastUnavailableReason = "dead"; + return directSendOrRefuse(url, input, init, "proxy respawn failed — model requests go direct (uncompressed)"); } throw err; } @@ -1334,5 +1517,11 @@ export function _resetForTest(opts: { anchor?: typeof globalThis.fetch } = {}): delete g[ACTIVE_STATE]; delete g[ACTIVE_STATE_SEQ]; signedDirectSeen.clear(); + respawnCooldownOverrideMs = undefined; resetFetchChainForTest(opts); } + +/** Test-only: override the #4516 cooldown-gated respawn retry cadence. */ +export function _setRespawnCooldownForTest(ms: number | undefined): void { + respawnCooldownOverrideMs = ms; +} diff --git a/src/agent/pi-native.ts b/src/agent/pi-native.ts index 277cdc7d4a..c48600a9c8 100644 --- a/src/agent/pi-native.ts +++ b/src/agent/pi-native.ts @@ -27,7 +27,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { ensureProxyRunning, LAUNCHER_DEFAULT_HOST } from "../launcher.js"; +import { ensureProxyRunning, LAUNCHER_DEFAULT_HOST, SPAWN_BUDGET_MS } from "../launcher.js"; import { piPackageStrings } from "../plugin-install.js"; import { createBiliPlugin, createPiDirectSendGuard } from "./pi.js"; import { isLegacyBcpEntry, markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, setNativeOriginWaiter, singleFlight } from "./native-bootstrap.js"; @@ -286,6 +286,12 @@ export function armNativePi(p: typeof plan): void { } else if (process.env.NODE_TEST_CONTEXT === undefined) { const start = singleFlight(bootstrap); state.respawn = start; + // #4516: a request may hold out for an IN-FLIGHT spawn under the + // launcher's own bring-up budget (+margin for health settle) instead of + // degrading at the generic 15s readiness cap — a slow cold start + // (#4515) must not refuse or direct-send a turn the proxy would have + // served had it finished starting a few seconds later. + state.spawnWaitMs = SPAWN_BUDGET_MS + 10_000; // #1243: the factory's before_provider_headers reads the proxy base from // BILLION_CONTEXT_PROXY, which bootstrap() writes asynchronously — a // one-shot's single header event fires before that. Publish the ready @@ -335,6 +341,7 @@ export function _resetNativeStateForTest(): void { state.onRoutedOriginObserved = undefined; state.respawn = undefined; state.onGiveUp = undefined; + state.spawnWaitMs = undefined; state.ready = Promise.resolve(undefined); _spawnForTest = undefined; delete process.env.BILLION_CONTEXT_PROXY; diff --git a/tests/issue4516-pi-direct-fallback.test.ts b/tests/issue4516-pi-direct-fallback.test.ts new file mode 100644 index 0000000000..a3f42d088f --- /dev/null +++ b/tests/issue4516-pi-direct-fallback.test.ts @@ -0,0 +1,279 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, test } from "node:test"; + +// #4516: the gateless lanes (pi/omp/opencode-V1) had NO size judgment on their +// degraded direct-fallback path — a 1.06M-token raw history rode straight to +// the provider when the proxy was down. This suite pins the three facets of +// the fix in the SHARED intercept: the local oversize refusal (with #4111 +// media-aware math, fail-open without a window header), the cooldown-gated +// respawn retry (no more direct-for-process-lifetime after a failed spawn), +// and the transition logging (every entry into / exit out of the degraded +// state lands in bili.log). Hermetic like native-intercept-selfheal.test.ts: +// XDG_STATE_HOME redirected so no event reaches a real log file. +const STATE_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4516-state-")); +const PREV_XDG_STATE_HOME = process.env.XDG_STATE_HOME; +process.env.XDG_STATE_HOME = STATE_ROOT; +const CHAIN_LOG_FILE = path.join(STATE_ROOT, "billion-context", "bili.log"); +after(() => { + if (PREV_XDG_STATE_HOME === undefined) delete process.env.XDG_STATE_HOME; + else process.env.XDG_STATE_HOME = PREV_XDG_STATE_HOME; + fs.rmSync(STATE_ROOT, { recursive: true, force: true }); +}); + +const UPSTREAM = "http://127.0.0.1:8199/v1/chat/completions"; +const PROXY = "http://127.0.0.1:40001"; +const ROUTED = `${PROXY}/bili/${UPSTREAM}`; + +function fakeFetch(sink: string[]) { + return (async (input: string | URL | Request, _init?: RequestInit) => { + sink.push(typeof input === "string" ? input : input instanceof URL ? input.href : (input as Request).url); + return new Response("{}", { status: 200 }); + }) as typeof fetch; +} + +function chatBody(textChars: number): string { + return JSON.stringify({ model: "gpt-test", messages: [{ role: "user", content: "x".repeat(textChars) }] }); +} + +function mediaBody(textChars: number, b64Chars: number): string { + return JSON.stringify({ + model: "gpt-test", + messages: [{ role: "user", content: [ + { type: "text", text: "x".repeat(textChars) }, + { type: "image_url", image_url: { url: `data:image/png;base64,${"A".repeat(b64Chars)}` } }, + ] }], + }); +} + +interface Ctx { + sink: string[]; + fetch: () => typeof fetch; + rearm: (v: typeof fetch) => void; + state: import("../src/agent/native-intercept.js").NativeInterceptState; +} + +/** Drive requests through a freshly installed intercept. The respawn stub + * fails by default (counts every fire); a test can reassign `state.respawn` + * mid-run to land a recovery, since the intercept reads it per request. */ +async function withFallback( + fn: (ctx: Ctx & { respawnCount: () => number }) => Promise, + opts: { origin?: string; readyTimeoutMs?: number; spawnWaitMs?: number } = {}, +): Promise<{ sink: string[]; log: string; result: T }> { + const saved = globalThis.fetch; + const sink: string[] = []; + let respawnCalls = 0; + const state: import("../src/agent/native-intercept.js").NativeInterceptState = { + origin: opts.origin, + ready: Promise.resolve(opts.origin), + readyTimeoutMs: opts.readyTimeoutMs, + spawnWaitMs: opts.spawnWaitMs, + respawn: async () => { + respawnCalls += 1; + return undefined; + }, + }; + const mod = await import("../src/agent/native-intercept.js"); + mod._resetForTest(); + try { + fs.rmSync(CHAIN_LOG_FILE, { force: true }); + } catch { + // first test of the file — nothing to clear + } + globalThis.fetch = fakeFetch(sink); + try { + assert.equal(mod.installNativeFetchIntercept(state), true); + const result = await fn({ + sink, + fetch: () => globalThis.fetch, + rearm: (v) => { globalThis.fetch = v; }, + state, + respawnCount: () => respawnCalls, + }); + return { sink, log: fs.existsSync(CHAIN_LOG_FILE) ? fs.readFileSync(CHAIN_LOG_FILE, "utf8") : "", result }; + } finally { + globalThis.fetch = saved; + mod._resetForTest(); + } +} + +test("#4516 oversize + window header + bootstrap failed → local 422, upstream never hit", async () => { + const { sink, result } = await withFallback(async ({ fetch }) => { + const res = await fetch()(UPSTREAM, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: chatBody(4_200_000), + }); + const body = await res.json() as Record; + return { status: res.status, gate: res.headers.get("x-bili-gate"), body }; + }); + assert.equal(result.status, 422); + assert.equal(result.gate, "oversize-refusal-fallback"); + assert.equal((result.body.error as { type: string }).type, "server_error"); + assert.equal((result.body.error as { code: string }).code, "bili_gate_oversize_refusal_fallback"); + assert.equal((result.body.error as { retryable: boolean }).retryable, true); + assert.match(String((result.body.error as { message: string }).message), /did not come up/); + // #4079 contract: the 500-token declaration is printed, but the local bar + // never assumes below 1M — an undersized declaration cannot false-refuse. + assert.match(String((result.body.error as { message: string }).message), /500 declared; the local bar never assumes below 1,000,000/); + assert.deepEqual(sink, [], "the provider must never see an oversized direct send"); +}); + +test("#4516 same refusal, anthropic wire shape (/messages)", async () => { + const { result } = await withFallback(async ({ fetch }) => { + const res = await fetch()("http://127.0.0.1:8199/v1/messages", { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: chatBody(4_200_000), + }); + return { status: res.status, body: await res.json() as Record }; + }); + assert.equal(result.status, 422); + assert.equal(result.body.type, "error"); + assert.equal((result.body.error as { type: string }).type, "invalid_request_error"); +}); + +test("#4516 fits the window → still goes direct (the refusal must not over-suppress)", async () => { + const { sink, result } = await withFallback(async ({ fetch }) => { + const res = await fetch()(UPSTREAM, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: chatBody(200), + }); + return res.status; + }); + assert.equal(result, 200); + assert.deepEqual(sink, [UPSTREAM], "a fitting body keeps #1117-style direct semantics"); +}); + +test("#4516 no window header → fail open to direct (never a false refusal from missing data)", async () => { + const { sink, result } = await withFallback(async ({ fetch }) => { + const res = await fetch()(UPSTREAM, { method: "POST", headers: { "content-type": "application/json" }, body: chatBody(4_200_000) }); + return res.status; + }); + assert.equal(result, 200); + assert.deepEqual(sink, [UPSTREAM]); +}); + +test("#4516 media-heavy body whose CORE TEXT fits → no false refusal (#4111 direction)", async () => { + const { sink, result } = await withFallback(async ({ fetch }) => { + const res = await fetch()(UPSTREAM, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: mediaBody(100, 4_200_000), + }); + return res.status; + }); + assert.equal(result, 200); + assert.deepEqual(sink, [UPSTREAM], "media mass is excluded from the doom math"); +}); + +test("#4516 media-heavy body whose core text is STILL over → refused", async () => { + const { sink, result } = await withFallback(async ({ fetch }) => { + const res = await fetch()(UPSTREAM, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: mediaBody(4_200_000, 4_200_000), + }); + return res.status; + }); + assert.equal(result, 422); + assert.deepEqual(sink, []); +}); + +test("#4516 self-heal: failed bootstrap no longer means direct-for-process-lifetime", async () => { + const { sink, log, result } = await withFallback(async ({ fetch, state, respawnCount }) => { + const mod = await import("../src/agent/native-intercept.js"); + mod._setRespawnCooldownForTest(30); + // Request 1: the armed stub fails → degraded direct send. + const first = await fetch()(UPSTREAM, { method: "POST", headers: { "content-type": "application/json" }, body: chatBody(100) }); + assert.equal(first.status, 200); + assert.equal(respawnCount(), 1); + // The proxy comes back on the NEXT attempt (counted here — the + // replacement stub no longer goes through the harness counter). + let recoveredFires = 0; + state.respawn = async () => { + recoveredFires += 1; + return PROXY; + }; + await new Promise((r) => setTimeout(r, 40)); + const second = await fetch()(UPSTREAM, { method: "POST", headers: { "content-type": "application/json" }, body: chatBody(100) }); + assert.equal(second.status, 200); + return respawnCount() + recoveredFires; + }); + assert.equal(sink[0], UPSTREAM, "first request went direct while the spawn was dead"); + assert.ok(sink.includes(ROUTED), `recovered request must route through bili, got ${JSON.stringify(sink)}`); + assert.equal(result, 2, "one respawn attempt per cooldown window, then recovery"); + assert.match(log, /proxy not ready — model requests go direct \(uncompressed\)/); + assert.match(log, /proxy recovered — model requests resume compression \(was direct\)/); +}); + +test("#4516 cooldown bounds respawn retries: concurrent requests share ONE attempt", async () => { + const { result } = await withFallback(async ({ fetch, respawnCount }) => { + const mod = await import("../src/agent/native-intercept.js"); + mod._setRespawnCooldownForTest(5000); + await Promise.all([ + fetch()(UPSTREAM, { method: "POST", headers: { "content-type": "application/json" }, body: chatBody(100) }), + fetch()(UPSTREAM, { method: "POST", headers: { "content-type": "application/json" }, body: chatBody(100) }), + ]); + return respawnCount(); + }); + assert.equal(result, 1, "the second concurrent request must NOT stack another spawn attempt"); +}); + +test("#4516 slow cold start: an in-flight spawn outliving the readiness cap is waited out, not refused (#4515 shape)", async () => { + // Models the field report: bootstrap takes longer than the generic 15s + // cap but finishes inside the owner's bring-up budget. Pre-fix the turn + // would have been refused (or sent direct into a guaranteed upstream + // rejection) at the cap; post-fix it waits out the spawn and routes. + const slowOrigin = new Promise((resolve) => setTimeout(() => resolve(PROXY), 120)); + const { sink, result } = await withFallback(async ({ fetch, state }) => { + // Bootstrap already in flight at arm time (pending past the 50ms + // readiness cap); the retry path must wait out the SAME in-flight + // attempt under spawnWaitMs instead of refusing at the cap. + state.ready = slowOrigin; + state.respawn = () => slowOrigin; + const res = await fetch()(UPSTREAM, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: chatBody(4_200_000), + }); + return res.status; + }, { readyTimeoutMs: 50, spawnWaitMs: 300 }); + assert.equal(result, 200, "the oversized turn must route once the in-flight spawn lands"); + assert.deepEqual(sink, [ROUTED], "waited out the spawn — nothing went direct"); +}); + +test("#4516 transition logging: repeated degraded sends log ONCE per transition", async () => { + const { log } = await withFallback(async ({ fetch }) => { + const mod = await import("../src/agent/native-intercept.js"); + mod._setRespawnCooldownForTest(5000); + for (let i = 0; i < 3; i++) { + const res = await fetch()(UPSTREAM, { method: "POST", headers: { "content-type": "application/json" }, body: chatBody(100) }); + assert.equal(res.status, 200); + } + return undefined; + }); + const directLines = log.split("\n").filter((l) => l.includes("model requests go direct (uncompressed)")).length; + assert.equal(directLines, 1, `degraded entry logs exactly once, got ${directLines}`); +}); + +test("#4516 dead-proxy wording: after holding an origin, the refusal says it DIED (not 'did not come up')", async () => { + const dyingFetch = (async (input: string | URL | Request, init?: RequestInit) => { + throw new TypeError("fetch failed"); + }) as typeof fetch; + const { sink, result } = await withFallback(async ({ fetch, rearm }) => { + rearm(dyingFetch); + const res = await fetch()(UPSTREAM, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin-context-window": "500" }, + body: chatBody(4_200_000), + }); + return { status: res.status, body: await res.json() as Record }; + }, { origin: PROXY }); + assert.equal(result.status, 422); + assert.match(String((result.body.error as { message: string }).message), /died and no replacement came up/); + assert.deepEqual(sink, [], "the direct fallback send is refused before touching the provider"); +}); diff --git a/tests/native-intercept.test.ts b/tests/native-intercept.test.ts index 759f0f5b3b..1ec7aad8a3 100644 --- a/tests/native-intercept.test.ts +++ b/tests/native-intercept.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, liveStateSlotEffect, noteRoutedOrigin, observeRoutedOrigin, _resetForTest, type NativeInterceptState } from "../src/agent/native-intercept.ts"; +import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, liveStateSlotEffect, noteRoutedOrigin, observeRoutedOrigin, _resetForTest, _setRespawnCooldownForTest, type NativeInterceptState } from "../src/agent/native-intercept.ts"; import { makeRecordingFetch, runHotReplace, type RecordedCall } from "./helpers/hot-replace.ts"; test("isModelApiUrl: matches model-API endpoint shapes", () => { @@ -589,13 +589,25 @@ test("install: failed respawn degrades to a direct send and fires onGiveUp", asy assert.equal(state.origin, undefined); assert.equal(respawns, 1); assert.equal(giveUps, 1); - // The degrade is permanent for the session: later model requests go - // direct without re-entering the respawn path. + // #4516: the degrade is NO LONGER permanent — a later model request + // re-enters the owner's bootstrap as a cooldown-gated self-heal retry + // (one attempt per window until the proxy comes back). Allow an + // immediate retry and pin the positive direction of that gate. + _setRespawnCooldownForTest(0); const res2 = await globalThis.fetch("http://127.0.0.1:8199/v1/messages"); assert.equal(res2.status, 200); assert.equal(calls[2], "http://127.0.0.1:8199/v1/messages"); assert.deepEqual(dispatches, ["rewrite", "direct", "direct"]); - assert.equal(respawns, 1); + assert.equal(respawns, 2); + assert.equal(giveUps, 1); + // Negative direction (#3617): inside the cooldown window the + // self-heal must NOT stack another respawn attempt. + _setRespawnCooldownForTest(1_000_000); + const res3 = await globalThis.fetch("http://127.0.0.1:8199/v1/messages"); + assert.equal(res3.status, 200); + assert.equal(calls[3], "http://127.0.0.1:8199/v1/messages"); + assert.deepEqual(dispatches, ["rewrite", "direct", "direct", "direct"]); + assert.equal(respawns, 2); assert.equal(giveUps, 1); } finally { globalThis.fetch = saved; From 5b8424d2c416247b3315c6f281a8f1f656fd37c1 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 24 Oct 2026 23:27:22 +0800 Subject: [PATCH 02/13] refactor(#4546): use restamped.init consistently in the routed retry send Review nit from the #4546 review: the recovery-retry path built `restamped` but dispatched with `stamped.init` (and read outgoing headers off `stamped.init`). `withHeaders` returns an `.init` that is a pure function of only (`init`, `extra`) - never of its first argument - and both calls pass the identical `init` + `routedExtra`, so `stamped.init` == `restamped.init` in every branch. Provably no behavior change; this just makes both dispatch arguments come from the same `restamped` binding for readability. --- src/agent/native-intercept.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 09aaec0460..94f22fecdc 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -1269,9 +1269,9 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): const retried = `${again}${u.pathname}${u.search}`; const restamped = withHeaders(makeTarget(retried), init, routedExtra); state.onDispatch?.(retried, "retry"); - const resp = await send(restamped.input, stamped.init); + const resp = await send(restamped.input, restamped.init); try { - state.onResponse?.(url, resp, outgoingHeadersOf(restamped.input, stamped.init)); + state.onResponse?.(url, resp, outgoingHeadersOf(restamped.input, restamped.init)); } catch { // observability hook only — never break the response } From 18f514667dc9988068c325665601b4c3bd0e2a08 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 23:02:14 +0800 Subject: [PATCH 03/13] test: re-pin storage-lane proofs onto the #4316 one-time ledger alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4316 PR3/3 (restampRefsToLedger) re-homes citation numbers once on every full-mode cold load; goldens pinning exact ref numbers are born-red. Re-pin the 7 storage-lane tests (engine-proofs x6 + orphan-recovery x1) modulo that renumbering — zero production code touched. - golden-storage-drive.ts: per-turn citation-normalized digest (shaCiteNorm) and state digest modulo stripOneTimeAlignment (stateDigestAligned); the deep-sort helper is hoisted into golden-canonical.ts for sharing. - corrupt-index-child.ts: new stripOneTimeAlignment normalizer — strips the refLedgerAligned stamp and re-homes refs onto canonical raw-id order so a clean full-mode read and the legacy fallback load compare equal. - storage-engine-proofs.test.ts: wire legs assert shaCiteNorm equality vs the legacy restart leg and the committed goldens; state legs assert stateDigestAligned; rollback leg now spells BILI_STORAGE_UNIFIED=legacy explicitly (#3818 flip: absent resolves to full, deleting the env would vacate the legacy path). - storage-orphan-recovery.test.ts: pins the content-addressable byRaw->byRef round-trip instead of a counter-era m00003 literal. - golden/storage-render/*.json regenerated with shaCiteNorm/stateDigestAligned. Converges the master-red storage-lane family (#4599 plan A, owner directive). --- tests/corrupt-index-child.ts | 52 +++++++++++++++++++++- tests/golden-canonical.ts | 16 +++++++ tests/golden-storage-drive.ts | 46 ++++++++++++++----- tests/golden/storage-render/anthropic.json | 46 ++++++++++++------- tests/golden/storage-render/chat.json | 46 ++++++++++++------- tests/golden/storage-render/google.json | 46 ++++++++++++------- tests/golden/storage-render/responses.json | 46 ++++++++++++------- tests/storage-engine-proofs.test.ts | 43 ++++++++++++------ tests/storage-orphan-recovery.test.ts | 7 ++- 9 files changed, 259 insertions(+), 89 deletions(-) diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 2d59e4a869..32000ce1ca 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -43,7 +43,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; -import { canonicalize, sha256 } from "./golden-canonical.ts"; +import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; // --------------------------------------------------------------------------- // Shared with the parent test files (moved verbatim, single definition). @@ -81,6 +81,50 @@ export function viewStateDigest(view: ReturnType): string }))); } +/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a + * ONE-TIME ledger alignment — citation numbers are re-homed onto ledger + * sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped. + * Cross-mode / cross-read-path "identical state" proofs must therefore compare + * the state modulo that one-time renumbering: this strips the alignment stamp + * and re-homes every ref number to a canonical order (k-th smallest raw id -> + * m0000k), so two views differing ONLY in which numbering scheme produced + * their refs hash identically under viewDigest/viewStateDigest. Object key + * order is normalized too: different hydration paths build identical content + * under different insertion orders, and canonicalize preserves them. Pure + * function — returns a new object, input untouched. */ +export function stripOneTimeAlignment(view: T): T { + const v = view as { state?: unknown; blockContents?: unknown }; + if (!v.state || typeof v.state !== "object") return view; + const byRaw = (v.state as { messageRefs?: { byRaw?: Record } }).messageRefs?.byRaw; + const remap = new Map(); + if (byRaw) { + for (const [raw, ref] of Object.entries(byRaw).sort((a, b) => (a[0] < b[0] ? -1 : 1))) { + remap.set(ref, `m${String(remap.size + 1).padStart(5, "0")}`); + } + } + // The JSON pass below flattens Maps — pull blockContents out while it is + // still a Map, renumber its keys afterwards, rebuild it. Everything else + // in these views is JSON-native. + const bcIsMap = v.blockContents instanceof Map; + let bcEntries: Array<[string, string]> | null = null; + let source: unknown = view; + if (bcIsMap) { + bcEntries = [...(v.blockContents as Map).entries()]; + source = { ...(v as Record), blockContents: undefined }; + } + let json = JSON.stringify(canonicalize(source)); + if (remap.size > 0) { + // Single-pass alternation: sequential per-token replacement would chain + // (a target that is also another source gets re-mapped on the next pass). + const re = new RegExp(`\\b(${[...remap.keys()].join("|")})\\b`, "g"); + json = json.replace(re, (tok) => remap.get(tok)!); + } + const out = JSON.parse(json) as (T & { state?: Record; blockContents?: unknown }) & Record; + if (out.state && "refLedgerAligned" in out.state) delete out.state["refLedgerAligned"]; + if (bcEntries) out.blockContents = new Map(bcEntries.map(([k, val]) => [remap.get(k) ?? k, val])); + return sortObjectKeys(out) as T; +} + export interface StorageWire { requested: string; effective: string; @@ -277,8 +321,12 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise { // Regression (#2781): the failed open must have CLOSED its handle. await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-moved"); + // #4316 PR3/3: the clean full-mode read carried the one-time ledger + // alignment; the legacy fallback over the corrupt index did not. The + // parent passes its digest already modulo that alignment (single + // definition), so strip it here too before comparing. await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () => - assert.equal(viewStateDigest(toLegacyLike(reloaded)), expectedStateDigest, + assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest, "corrupt-index fallback state digest drift vs clean full-mode read")); // Writes must keep working (legacy path) — no throw, file refreshed. diff --git a/tests/golden-canonical.ts b/tests/golden-canonical.ts index 09f491f2a7..6736b33311 100644 --- a/tests/golden-canonical.ts +++ b/tests/golden-canonical.ts @@ -4,6 +4,22 @@ export { canonicalize, sha256 } from "../src/storage/canonical.ts"; import { canonicalize, sha256 } from "../src/storage/canonical.ts"; +/** Deterministic key order for cross-read-path digest comparisons: two views + * built by different hydration paths carry identical content under different + * map insertion orders, and canonicalize preserves insertion order — sorting + * here makes the digests path-independent. Maps/Sets pass through untouched + * (callers that need them iterable keep them intact; canonicalize flattens + * them downstream). */ +export function sortObjectKeys(v: unknown): unknown { + if (Array.isArray(v)) return v.map(sortObjectKeys); + if (v && typeof v === "object" && !(v instanceof Map) && !(v instanceof Set)) { + const out: Record = {}; + for (const k of Object.keys(v as Record).sort()) out[k] = sortObjectKeys((v as Record)[k]); + return out; + } + return v; +} + /** Canonical JSON → sha256, with a final pass of STRING-level environment * normalization (ports, session ids embedded inside text fields). */ function digestOf(value: unknown, extra?: { normalize?: (s: string) => string }): string { diff --git a/tests/golden-storage-drive.ts b/tests/golden-storage-drive.ts index a6b14205f8..4d65412aca 100644 --- a/tests/golden-storage-drive.ts +++ b/tests/golden-storage-drive.ts @@ -16,10 +16,11 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; import { once } from "node:events"; -import { canonicalize, sha256 } from "./golden-canonical.ts"; +import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; import { defaultConfig } from "acp-kernel"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { stripOneTimeAlignment } from "./corrupt-index-child.ts"; import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts"; import type { ProxyOptions } from "../src/config.ts"; @@ -110,7 +111,7 @@ const GROW_TURNS = 20; // hard cap for the grow loop const SUMMARY_MARKER = "[Compressed conversation section]"; const EDIT_SENTINEL = "GOLDEN-EDIT-SENTINEL"; -interface GoldenTurn { i: number; sha: string; bytes: number } +interface GoldenTurn { i: number; sha: string; bytes: number; shaCiteNorm: string } interface GoldenRecord { wire: Wire; turns: GoldenTurn[]; @@ -118,6 +119,7 @@ interface GoldenRecord { editedAt: number; // body index after the mid-history edit summaryFrom: number; // first body index carrying the fold summary stateDigest: string; // sha256 of the canonicalized persisted session + stateDigestAligned: string; // #4316 PR3/3: same, modulo the one-time ledger alignment sectionDigests: Record; // drift localization (same normalization) stateKeyDigests: Record; // per-top-level-state-key digests stateShape: { // human-readable facts baked into the snapshot @@ -169,6 +171,18 @@ function canonicalBody(body: string): string { .replace(/msg-fix-[0-9a-f]+/g, "msg-fix-"); } +// #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation numbers +// once (one-time ledger alignment) — that renumbering is the ONLY permitted +// wire delta vs the legacy restart. Wire-parity proofs therefore pin a second +// digest per turn with every citation number masked to : equality says +// "only the citation numbers may differ", everything else stays byte-exact. +// All m\d{5} tokens are masked (not just tag payloads) because summaries cite +// numbers in prose too; user content is identical across legs, so the wider +// mask cannot hide a real divergence. +function citeNormBody(body: string): string { + return body.replace(/\bm\d{5}\b/g, ""); +} + /** Timestamps (ms-epoch magnitude) and UUIDs become placeholders; everything * else — refs, block ids, coverage sets, stats — must be byte-stable. */ // canonicalize/sha256 live in ./golden-canonical.ts (shared with the storage-corpus pins). @@ -585,15 +599,7 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: // order, so a harmless field-reorder commit (#3093) red all four // goldens. Deep-sort object keys harness-side only; array order is // preserved (foldAnchorOrder & co are ordered contracts). - const sortObjectKeys = (v: unknown): unknown => { - if (Array.isArray(v)) return v.map(sortObjectKeys); - if (v && typeof v === "object") { - const o: Record = {}; - for (const k of Object.keys(v as Record).sort()) o[k] = sortObjectKeys((v as Record)[k]); - return o; - } - return v; - }; + const lastView = JSON.stringify(canonicalize(sortObjectKeys({ state: session.state, blockContents: [...session.blockContents.entries()], @@ -606,6 +612,18 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: .replace(/127\.0\.0\.1:\d+/g, "127.0.0.1:") .split(JSON.stringify(sessionId).slice(1, -1)).join(""); const stateDigest = sha256(lastView); + // #4316 PR3/3: parallel digest modulo the one-time ledger alignment — + // cross-mode proofs compare states that may differ only in ref numbering. + const stateDigestAligned = sha256( + JSON.stringify(canonicalize(sortObjectKeys(stripOneTimeAlignment({ + state: session.state, + blockContents: [...session.blockContents.entries()], + stats: session.stats, + metadata: session.metadata, + })))) + .replace(/127\.0\.0\.1:\d+/g, "127.0.0\.1:") + .split(JSON.stringify(sessionId).slice(1, -1)).join(""), + ); // Localization aid for cross-machine digest drift: per-section and // per-top-level-state-key digests under the SAME normalization, so a // CI failure message pinpoints the divergent subtree without needing @@ -629,11 +647,15 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: const active = st.blocks?.filter((b) => b.active) ?? []; return { wire, - turns: bodies.map((b, i) => ({ i, sha: sha256(canonicalBody(b)), bytes: Buffer.byteLength(b, "utf8") })), + turns: bodies.map((b, i) => { + const cb = canonicalBody(b); + return { i, sha: sha256(cb), bytes: Buffer.byteLength(b, "utf8"), shaCiteNorm: sha256(canonicalBody(citeNormBody(b))) }; + }), foldAt, editedAt, summaryFrom: bodies.findIndex((b) => b.includes(SUMMARY_MARKER)), stateDigest, + stateDigestAligned, sectionDigests, stateKeyDigests, stateShape: { diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 5afa14b2c4..f2c7ad03fa 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -4,83 +4,99 @@ { "i": 0, "sha": "daf2182687948ef53790608f8fe13f21f457552bd56c61e914699d71793a95d0", - "bytes": 22287 + "bytes": 22287, + "shaCiteNorm": "37a0f1ef30156301cae88e2dece8a03f8ba557440925d3f8fd4b6f654e9880aa" }, { "i": 1, "sha": "2894d8949694f953aef9b0a7d4c4ad03b3633b8cd90886852109468d10e72b45", - "bytes": 30997 + "bytes": 30997, + "shaCiteNorm": "ae251c0c242448c1d87f417194f38d42c901e563ca0234a616e22254d8d29352" }, { "i": 2, "sha": "911f0c635e145362329fef63920e7b4802f2f9c9528062173e867ad891832873", - "bytes": 36896 + "bytes": 36896, + "shaCiteNorm": "efbc1fcc13a7968e0dd8120bc1dc0e04b24d0aac997700fe52dd414e0270c351" }, { "i": 3, "sha": "425dee6b9938613b65f9c944aea28228eda5679915fd39597196876dda2915f6", - "bytes": 45569 + "bytes": 45569, + "shaCiteNorm": "0d5a15e5216f11dd47816ff0bc336532ffdaec00fecad51ff2c9995a4bb00254" }, { "i": 4, "sha": "e62cbddd7d1facbcf2efe3c781c09b14b5cf096067b7d7ae4818ee7a4639e0b6", - "bytes": 51431 + "bytes": 51431, + "shaCiteNorm": "43615d160671fcbd9408a80e0779f6cf122fdbde81b901e35a277d8e6ee8495a" }, { "i": 5, "sha": "03a6265741bdb3f74788ae479f5b1505e552e6082009dd3419605dff8143b5da", - "bytes": 60104 + "bytes": 60104, + "shaCiteNorm": "66411dc16ffa93bb1fbc28b167302372c7b170af10bfca6c95ef9466cf4b8628" }, { "i": 6, "sha": "c7f62d7a595006a08aaa71427f0bcdb3b620c8eac104723bc6ace7516aa3321c", - "bytes": 65966 + "bytes": 65966, + "shaCiteNorm": "c9d9ff30521f6d1fa5e3e94f9ed778a1491224fc6440cd8bc3dde9e6bd7f77e6" }, { "i": 7, "sha": "edb9fadc1d20813a24f45ea2d1e2ab0d92972d1cee22d45ac278d9dd1c17bd4c", - "bytes": 74639 + "bytes": 74639, + "shaCiteNorm": "4e4c14113531fb4152983af6c3596dd3944f30ca0ab4e720e92a31bd7a253a9d" }, { "i": 8, "sha": "d2be70c3953a492bf1b974ff4922001d74d0d61012446c59e732fec18852d713", - "bytes": 80501 + "bytes": 80501, + "shaCiteNorm": "ca749fa6f80100f404d828594314c8823dc2900aedb089264a057e72f54536fa" }, { "i": 9, "sha": "2ddad1b2186f7f0444543f1b72b2f4fe3dcfe6d8cf709211a54e820b7c3a6a4a", - "bytes": 49528 + "bytes": 49528, + "shaCiteNorm": "8008f1d0bbe5e604ca7b1201e6d60e04ab7a10c117373eda0412308319f5c6b2" }, { "i": 10, "sha": "7da7b1f5aed39a2af9df234e8ad97fff4fe1cf49623a3adc4a3c45cd22989be0", - "bytes": 57070 + "bytes": 57070, + "shaCiteNorm": "09e0e277a54ce862f28c41a40d4b1261a8b8198240a10191986f07ddb1b9ad34" }, { "i": 11, "sha": "fca40b7b4dae14bfa94542cb8d802ce7b097b7b2946f987437db98a1cbc0385c", - "bytes": 62892 + "bytes": 62892, + "shaCiteNorm": "58178314a73fd15478a8d1cef6e6f3f535e216ff3a27d5d5c0af85f3cc480fe5" }, { "i": 12, "sha": "8b5ff77e47604fe58cbb19234306863e3ec754c1d51a7a354bab40ffb12ef59a", - "bytes": 77030 + "bytes": 77030, + "shaCiteNorm": "0afc5df7cd478a9a127da7cb011c61a20cbf35ebe31409daad93e0e9c2944f24" }, { "i": 13, "sha": "375a80af22ca3da9763155fd5394cff9066e0695490105830137cdcd9432f7d1", - "bytes": 82815 + "bytes": 82815, + "shaCiteNorm": "c16a407ec4983f77cd62792e46a093c839853077d4b52cd799b81adb38a1ca20" }, { "i": 14, "sha": "3f23c183e08dfe04772ea8cf1b5196f8ad92a57868896d55aafc7eb9300ed53a", - "bytes": 91464 + "bytes": 91464, + "shaCiteNorm": "415619a3a81f1aad4f2790ffc63eaeb3a0410e27027e77609a01f0fbd1471abe" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, "stateDigest": "6727f42bd91879a7aa370e1ce0c78963bcba0df92e50390d4a8011f8f0da334d", + "stateDigestAligned": "25463f79510522e0c0d8a39127f9e9b5cc4465fb7293705ab9bea05ed4672ccb", "sectionDigests": { "state": "f93dc1c876c672d4", "blockContents": "dc7d13d250c94bb4", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 46764f97db..3820301619 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -4,83 +4,99 @@ { "i": 0, "sha": "824973c12a04e4a9c6fc8a5b84555886597d1a41874e6b0da33a489fe5f9fe9b", - "bytes": 22397 + "bytes": 22397, + "shaCiteNorm": "40255e64416e7fbea076dfe1195d32eac811a33d6e48db44922f93bfd3dadc31" }, { "i": 1, "sha": "4261484dd0b52e37019fe840a217bf4940369c7cb3c6d1ac28cb6353d4a68a37", - "bytes": 30975 + "bytes": 30975, + "shaCiteNorm": "7630710b673b4aa2dd1a7aade6809f5a97bdcd525857a37e9ab253dffcb82f43" }, { "i": 2, "sha": "e8050d3ec8b0e3dd2f0ca67a4a0328a8e280e23b6f4a3e34380af58e8d38d678", - "bytes": 36787 + "bytes": 36787, + "shaCiteNorm": "ad6b3f4bda6cf0b9b779034ea1b0ceb715c7a763df6b33eab4d5df7862d84b14" }, { "i": 3, "sha": "73d69f8b8f1bd2899cfe2181b95bc2f3697695e9b85ad682d82f6cbd9e7355ff", - "bytes": 45365 + "bytes": 45365, + "shaCiteNorm": "d683426210363474b3723a9dfd4e0e58e9eaf8b747291106d0eb22b86d6b1466" }, { "i": 4, "sha": "d3a73b4a74f7975de837b7848f24d875866b1f2b0b15f4ef78064e3602da6665", - "bytes": 51177 + "bytes": 51177, + "shaCiteNorm": "7b8084737aabd65b5d9be87be1610819f658c14a632de7f3575b6fbf44860e0b" }, { "i": 5, "sha": "60ce463e424f124a2dc346479091cbd4b7a32c69696b798a1de6b1a3c0f3ef7b", - "bytes": 59755 + "bytes": 59755, + "shaCiteNorm": "7b96653787fc0adda16590f35f3cb70f47bbecfed5e8208bc9f9520f0c360330" }, { "i": 6, "sha": "1298d5f45146b76a42b5b75c712de8011fb091c5754f99d490a7df2e81ba2466", - "bytes": 65567 + "bytes": 65567, + "shaCiteNorm": "33e2e632f57a5aeec30801779d9e38fcc55247389ac83217e88e0c6500b6a783" }, { "i": 7, "sha": "b20457e685b4a1fecf0195f57b93e052a0d00a83feb8b1569ccb08d385663aa2", - "bytes": 74145 + "bytes": 74145, + "shaCiteNorm": "0c7a0b8156c69d5710d2e395b6b8c985878c9120de8ff6c8f1fbe957649b855d" }, { "i": 8, "sha": "a0ba24010c6faf10f11b66faca6f3aa915dde7ac006d66a434777cbd765328ca", - "bytes": 79957 + "bytes": 79957, + "shaCiteNorm": "180576856d481f74a2e1fd352cb1ee881644fe905fb3263f16b12c90a7f5b86a" }, { "i": 9, "sha": "f1e08e530a96ecb4996c0a63e840e3f374a82b095e285ffea94e70391eb32fe0", - "bytes": 49458 + "bytes": 49458, + "shaCiteNorm": "7f1a433d2f553cc122e877912c99464485726c3d0707c5ffd12acd2478475176" }, { "i": 10, "sha": "8f3e444b1951f67e4f86abf6e727162108ba39d389b1ee4c90eea69df15bada0", - "bytes": 56859 + "bytes": 56859, + "shaCiteNorm": "90d0ab02fca529d89c1e045ec2bb157872e5b70508975b6d1782fc6c0a525cc9" }, { "i": 11, "sha": "94a8e322c3c0822a18bccd6295c3d2bfb4c7961d897298361612f286fb6108fe", - "bytes": 62594 + "bytes": 62594, + "shaCiteNorm": "9ca56f6252eb19787f5577ed01833ea816d242299c4f7a3b24351b7fb767860e" }, { "i": 12, "sha": "2c0a965647d8832dc74ebe5edc8cfe59b67abc936714119e89c93410f985e1af", - "bytes": 76640 + "bytes": 76640, + "shaCiteNorm": "39c2ab72032ed2149fdbb66698cffc18f6e0d96f713fb37181e08cb0cadc993a" }, { "i": 13, "sha": "a7497d037a324f1fdb6f605db0b8161e4e183f1e2354bb68035fb9f1373ee0d0", - "bytes": 82375 + "bytes": 82375, + "shaCiteNorm": "001e2a63c5352cd3230a30bc82dd6f9fbb8c6421f42e1bc8dd87551f50b8ae9f" }, { "i": 14, "sha": "abd73cec691568d84e72384aa5facdc71c759eeafc865d62df7b4eca6683675a", - "bytes": 90929 + "bytes": 90929, + "shaCiteNorm": "0f6af62ddbc1c0a39009524d581f52f728e928d438e998a716be5d054a8c6609" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, "stateDigest": "e149c301f382438ee063a38f70dd794207b88e94bdf3c14c5191da0eeaa99bc9", + "stateDigestAligned": "fb6382dd388b7aac449a1cc02f8d975a02c307ac4d9e9a46305a9d1f3f420468", "sectionDigests": { "state": "8390f21336885863", "blockContents": "94a58f96d7c5cfc3", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ba999cadec..ba4fbe2cb7 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -4,83 +4,99 @@ { "i": 0, "sha": "45998d29279880c0b19002f5ca40489e1e9dfcbdb53b9203a5ffe868ea3e51b8", - "bytes": 20510 + "bytes": 20510, + "shaCiteNorm": "a80634db66b3e2f5c96a66d48a7baf753f089d0af76e1c7370e44a22be3e2c50" }, { "i": 1, "sha": "f61275d77406c502c11238e2764ea84c8bf03c8302856749b8f5b73572fb5331", - "bytes": 29065 + "bytes": 29065, + "shaCiteNorm": "bbb40518ce7a359b6fb4d9ccebf356a433dee955daf8676d937fc2b1e2c493e0" }, { "i": 2, "sha": "98b77dc7c79575218c8e52ddec9386d5db7a7d30587dd992cb64c18536fb7667", - "bytes": 34891 + "bytes": 34891, + "shaCiteNorm": "3c18b9c31b2c19266a4f006c368008aa6dd0dbacdeb1bc00c86f3a76cd8fd8b5" }, { "i": 3, "sha": "2f3024ee57f441a1c0d1b06d38ccf1f4c74ef297aa3d58622bc29f69c21c93d6", - "bytes": 43446 + "bytes": 43446, + "shaCiteNorm": "8e074da69eef79658d01adab2bed04d55a232eebf3e437c0a94b4afefb52bb8c" }, { "i": 4, "sha": "27f6f223e7682e0e0790a69e43c982c3fe0e505153be7fa4d3ec6cf4532ff2e2", - "bytes": 49272 + "bytes": 49272, + "shaCiteNorm": "63b04fd0418ab224c41ee6e9ee2068a73b973c8df07e4e7f2ffbe6e19373ec7a" }, { "i": 5, "sha": "6a7e23920335f91762a8ac72b177a1cd68538ce1872325b6aa9e3b44191f3ca4", - "bytes": 57827 + "bytes": 57827, + "shaCiteNorm": "723ccf3a211d8b750959b479939ded61e4c9818473bde3a7364eae85ca9ad5da" }, { "i": 6, "sha": "5b770ce0c9456af96904cd5b007fbb07fb4194f2636db8b6ad2602ef71d3834d", - "bytes": 63653 + "bytes": 63653, + "shaCiteNorm": "4c20a67e2e4df13e061f8f094b92774d2c53c60a0711b6456731b943e510eec4" }, { "i": 7, "sha": "9bab6f659c0dcd05e6576e6d47b1d1d2378a6ace1a7effec721958f6f7476cd8", - "bytes": 72208 + "bytes": 72208, + "shaCiteNorm": "c807dfe30868d78848a846469e99e7d40fe7b866717974886c30ba8def051063" }, { "i": 8, "sha": "c179b940520413ba2a0e267165a9914712c3a9ff3c045761d61da876aeccedd7", - "bytes": 78034 + "bytes": 78034, + "shaCiteNorm": "2091c5306aba0a16a47547029fe8dd6b57cc8e31b3c6a2abea7d9a8ba7b63839" }, { "i": 9, "sha": "1b1448a075964dae98ad3b09565d31590cc9fdadb4d37300c4460030f67990e6", - "bytes": 50620 + "bytes": 50620, + "shaCiteNorm": "6ef485631895d0a448bb8d52235b9f3a00ba15143b587ac7253d119c83c61441" }, { "i": 10, "sha": "eddf6ad2081f5eb1e4822bf9805a7953eff3241269ed5f3eec346909c9da5bf2", - "bytes": 58023 + "bytes": 58023, + "shaCiteNorm": "caec6185725a235e023982a9d7f6b4fcb284bb6a2e21b89bb8f2b7e36bb1bf5c" }, { "i": 11, "sha": "0ae5c9b7b31dbb0d52c1b4a157bcc08c549ff435229383ec33fb0225b3c93f4e", - "bytes": 63772 + "bytes": 63772, + "shaCiteNorm": "894e9c4f827fb398b7a6c9bc91d430633e51bd552585fb5086c2cf3c18d6097f" }, { "i": 12, "sha": "d9a62f5c48f849bdc9514ae4586f2204ef5ac624aaf97c50dbc8ed2c1bee9706", - "bytes": 77777 + "bytes": 77777, + "shaCiteNorm": "2f1449ede69c9ee7db22f39c49584f64d35a29541097d78f524a4f22c5998acc" }, { "i": 13, "sha": "f60319947e77097d08edfcc5fdddd46057c44596a30d1d6bb33c5af4bad5001f", - "bytes": 83526 + "bytes": 83526, + "shaCiteNorm": "ee505218214a0e58875762c06c79e717164b89cfe29cdd61630fdd1fecb41a92" }, { "i": 14, "sha": "275f3b3e103dbfbf545448f8c421254cc7b025f8c081d941cd39bef4a90023be", - "bytes": 92056 + "bytes": 92056, + "shaCiteNorm": "c434408819f41c9e943e1cd60b80189f1844ba2c8f11898043f158f8485aee9b" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, "stateDigest": "80e83935e5fa7b052a3cb0f1d641bd81cb6ee59f91f3c1f2f31c204de29368a5", + "stateDigestAligned": "5447c342a7565f933d3093ef7054b746f41cd3a80513b72159ef4f2889042c38", "sectionDigests": { "state": "2e1def3a3234afdc", "blockContents": "473264d9df8ba338", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 7004d90435..c957f569ce 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -4,83 +4,99 @@ { "i": 0, "sha": "57662dd5517499b0833b00df645aacd9ec25a1add3b6efa94703cc40006824a8", - "bytes": 22467 + "bytes": 22467, + "shaCiteNorm": "dd2d60a2354cb8bb76834fa946ad985964def531c400f15a9f0370ee948fd039" }, { "i": 1, "sha": "ade76466ddbffb25c3eba9e06896c110c147ce1daf7f1e231fe054d9d1832c4b", - "bytes": 31395 + "bytes": 31395, + "shaCiteNorm": "9a1ea902b7287c23566da611dbab379637998536dc37365d47cd8577fa5f6a8c" }, { "i": 2, "sha": "09ee3e5a15865e5cc5c10c28026c1501f565212134c64a03214fbf367f63b881", - "bytes": 37517 + "bytes": 37517, + "shaCiteNorm": "99c6597f2bc37b03f6935208eb4be452619d599ce21b06f118f15657c5494a0b" }, { "i": 3, "sha": "cd3e38d97e64660eb60f1e387a1e53de8723ae1aa916db4d0a7b8f7ec181c266", - "bytes": 46445 + "bytes": 46445, + "shaCiteNorm": "947f17b43bc46a8f1ad828172f394e003f499af7ca16196f907620662a3a7929" }, { "i": 4, "sha": "b8ff4f16ca15cc1b8589251a849e3e30ec7d7d82b92abc0bb262bd2f8422db38", - "bytes": 52567 + "bytes": 52567, + "shaCiteNorm": "c0ce07681a604168b19c4e585112d5644555b8283a7c92c620336d0ec1453806" }, { "i": 5, "sha": "70f002ab5e7c3301ec440fd6863b7635c38874842445794bf87229f783ce2124", - "bytes": 61495 + "bytes": 61495, + "shaCiteNorm": "20c8f20fe476b978fcb07d34f4311a2712e8ea04e829b8ee0912e7741d6f95d1" }, { "i": 6, "sha": "aa63e444c4b86e426d949d2030a2b2a4f84a1018ebb152f0dd429c9747f357b3", - "bytes": 67617 + "bytes": 67617, + "shaCiteNorm": "91c714ad3016518bef0e07f9a7d227c9adcd164b5c3b1e62ee24b20bc38524d2" }, { "i": 7, "sha": "4cef1346cac328fc82bd552007051118cc95751e62d4b3f10a4432297a6801e5", - "bytes": 76545 + "bytes": 76545, + "shaCiteNorm": "68db839f454054f55c426b3e173d9baa9d4120a6f963cfa4856d4bad2b11e318" }, { "i": 8, "sha": "adee16938078b6c6cac3ca423ffe427d2199493d6cbbbc01059b223de3bba077", - "bytes": 82667 + "bytes": 82667, + "shaCiteNorm": "f8ca121812fc632d1b2cb140ac8ec41ceeb4f82916256386aa308d2a4272b09f" }, { "i": 9, "sha": "a6cefd10bbb51de65d7c363ea6f299a2301f4d8104a5ce8b59f4de87e711130d", - "bytes": 50498 + "bytes": 50498, + "shaCiteNorm": "266499f0528988fed9680ffef231be1162e6501857511a3de6a5f95ba1564256" }, { "i": 10, "sha": "ed8590ef249a0756d077fcea24230dd83a7f805a03da6f91457c607b27351c5e", - "bytes": 58283 + "bytes": 58283, + "shaCiteNorm": "5ed641acfb444fed79d86b5104b176d11250d309a649a2230ba2f3f08950453c" }, { "i": 11, "sha": "461eb77a6cc41abbf8046fb3dc5858f18c8d848a8dff4a2c888a38a1d16df707", - "bytes": 64328 + "bytes": 64328, + "shaCiteNorm": "c674b6af035c1b786fd0f47c0c33abc4179337e3df595391f3fcf9b6973ef531" }, { "i": 12, "sha": "b356956d375e951d2d5a05fe55c1e28b09c616f9b2d7eca6fea5e62924dbe53e", - "bytes": 78746 + "bytes": 78746, + "shaCiteNorm": "f8d20e84c64dfd8e224496a6feb949d16edc9de40f5100243d0514ec86eeddec" }, { "i": 13, "sha": "1b3d93417fce2ba8f70c52d0425626398a571d7afbd287ffb89a6ba7ae37bd29", - "bytes": 84794 + "bytes": 84794, + "shaCiteNorm": "b24c9b065471e5a264cb73d58b2383d9f8fc5b4feeb1f7d0fea05aeb96d1ff01" }, { "i": 14, "sha": "9c82e4523129ef8df61cf7622bb637c7379cce153c78e3814bd3000853034323", - "bytes": 93703 + "bytes": 93703, + "shaCiteNorm": "cd5c51970f832bcaa5f27406e353c8e67a53706c2bc8249e35634b6339dc893a" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, "stateDigest": "08526d3552fca7ac7552cd16a221893a652fbe740c9b8dccac6db57b3d6302ee", + "stateDigestAligned": "e8c52306206ba77ee93bce2062141e90e47675e03982c51f69ff42f4c6beffd4", "sectionDigests": { "state": "db313e8eba2d65fc", "blockContents": "b6701268a0bdcd8a", diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 118ca0bde7..d9eb3334b2 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -11,7 +11,11 @@ * render goldens, both with a mid-conversation restart * and with the legacy file DELETED at the restart (the * index alone carries the session — the migration's - * end-state promise). + * end-state promise). #4316 PR3/3 (380ef4dac): a + * full-mode cold load additionally re-homes citation + * numbers once (one-time ledger alignment) — comparisons + * run modulo that renumbering (citation-normalized + * digests), which is the ONLY permitted delta. * STALE REJECT — a multi-instance stale write that the guarded build * rejects must never reach index.db; the converged disk * record is what gets ingested. @@ -45,7 +49,7 @@ import { planIngest, loadLegacyView, toLegacyLike, type LegacySessionLike } from import { canonicalize, sha256 } from "./golden-canonical.ts"; import { driveWire, WIRES, type Wire } from "./golden-storage-drive.ts"; import { rmrf } from "./tmp-rm.ts"; -import { ENGPROOF_CHECKS, spawnCorruptIndexChild, viewDigest, viewStateDigest } from "./corrupt-index-child.ts"; +import { ENGPROOF_CHECKS, spawnCorruptIndexChild, stripOneTimeAlignment, viewDigest, viewStateDigest } from "./corrupt-index-child.ts"; process.env.BILI_PERSIST_ZSTD = "0"; // #3400: pin the legacy wire shape. This suite proves STORAGE restart @@ -145,11 +149,12 @@ function unifiedRows(store: UnifiedStore, id: string): ReturnType { + test(`storage engine: full-mode restart is invisible on the ${wire} wire (== legacy restart / committed goldens, modulo the #4316 one-time citation renumbering)`, { timeout: 240_000 }, async () => { const golden = JSON.parse(fs.readFileSync(path.join(RENDER_ROOT, `${wire}.json`), "utf8")) as { - turns: Array<{ i: number; sha: string }>; + turns: Array<{ i: number; sha: string; shaCiteNorm: string }>; foldAt: number; editedAt: number; summaryFrom: number; stateDigest: string; + stateDigestAligned: string; stateShape: Record; }; // The restart oracle: a legacy-only process restart. One caveat is @@ -162,16 +167,18 @@ for (const wire of WIRES) { const record = await driveWire(wire as Wire, `golden-${wire}`, { storageUnified: "full", reloadAfterEdit: leg }); assert.equal(record.turns.length, legacy.turns.length, `${wire}/${leg}: body count drifted vs legacy restart`); for (let i = 0; i < legacy.turns.length; i++) { - assert.equal(record.turns[i]!.sha, legacy.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs legacy restart`); - assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs committed goldens (restart must not change the wire)`); + // #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation + // numbers once (one-time ledger alignment) — the ONLY permitted + // wire delta. Citation-normalized digests must match turn-for-turn. + assert.equal(record.turns[i]!.shaCiteNorm, legacy.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs legacy restart`); + assert.equal(record.turns[i]!.shaCiteNorm, golden.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs committed goldens (restart-inherent bs-field loss aside)`); } assert.equal(record.foldAt, legacy.foldAt, `${wire}/${leg}: fold landmark drifted`); assert.equal(record.editedAt, legacy.editedAt, `${wire}/${leg}: edit landmark drifted`); assert.equal(record.summaryFrom, legacy.summaryFrom, `${wire}/${leg}: summary carrier landmark drifted`); - assert.equal(record.stateDigest, legacy.stateDigest, `${wire}/${leg}: persisted state digest drift vs legacy restart`); + assert.equal(record.stateDigestAligned, legacy.stateDigestAligned, `${wire}/${leg}: persisted state drifts beyond the one-time ledger alignment vs legacy restart`); + assert.equal(record.stateDigestAligned, golden.stateDigestAligned, `${wire}/${leg}: persisted state drifts beyond the one-time ledger alignment vs committed goldens`); assert.deepEqual(record.stateShape, legacy.stateShape, `${wire}/${leg}: state shape drift`); - assert.deepEqual(record.sectionDigests, legacy.sectionDigests, `${wire}/${leg}: state section digests drift`); - assert.deepEqual(record.stateKeyDigests, legacy.stateKeyDigests, `${wire}/${leg}: state key digests drift`); } }); } @@ -293,20 +300,25 @@ test("storage engine: rollback contract — unset the env and the legacy read pa // (The load-path digest is the stable oracle: an in-memory session vs // a loaded one differs benignly by the storageMode stamp + one-time // tail normalization, identically under both paths.) + // #3818 flip: absent now resolves to FULL, so the legacy read path has + // to be spelled out explicitly (the knob's one-line escape hatch) — + // merely deleting the env would read unified and vacate this leg. const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); storeFull.closeQuietly(); assert.ok(viaFull, "full-mode reload returned null"); const digestFull = viewDigest(toLegacyLike(viaFull)); - delete process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "legacy"; const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); const legacy = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); store2.closeQuietly(); assert.ok(legacy, "legacy-only reload returned null after rollback"); // #2960 ⑤: cross-mode equality is on the STATE proper — full mode // serves the raw table tail, legacy the envelope's folded view. - assert.equal(viewStateDigest(toLegacyLike(legacy)), viewStateDigest(toLegacyLike(viaFull)), "rollback: legacy-only load state digest drift vs full-mode load"); + // #4316 PR3/3: modulo the one-time ledger alignment (full-side cold + // load re-homes citation numbers; legacy keeps counter numbers). + assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(legacy))), viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))), "rollback: legacy-only load state digest drift vs full-mode load"); assert.ok(viaFull.lastMessages && viaFull.lastMessages.length > 0, "⑤: full-mode load serves the table tail"); assert.ok(legacy.lastMessages && legacy.lastMessages.length > 0, "⑤: rollback keeps the envelope tail"); const disk = findWrittenRecord(dir, "corpus-folded"); @@ -381,7 +393,10 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // executed in the child, verified in the parent). The child also runs // the #2781 mid-flow rename guard plus a post-close guard, so future // handle-leak drift trips there by name instead of at teardown. - const stateDigestClean = viewStateDigest(toLegacyLike(viaFull)); + // #4316 PR3/3: expected digest computed modulo the one-time ledger + // alignment — the clean read above already carried it in memory, and + // the child's fresh load applies it again; both sides must normalize. + const stateDigestClean = viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))); const childEnv: NodeJS.ProcessEnv = { ...process.env, BILI_STORAGE_UNIFIED: "full", @@ -509,5 +524,5 @@ test("storage engine: no SQLite engine -> SessionStore runs legacy-only, writes // Engine note: these legs run on the process-default engine; the corpus // suites (storage-unified.test.ts) cover BOTH engines at the state level, -// and the wire legs here assert against the same committed goldens the -// legacy baseline produced. +// and the wire legs here assert citation-normalized equality against the same +// committed goldens the legacy baseline produced. diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index c63f3f63fb..9fd6f13b0a 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -69,7 +69,12 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", assert.equal(recovered.stats.requests, 876); assert.equal(recovered.state.nextBlockId, 4); assert.deepEqual(recovered.state.blocks.map(b => b.blockId), ["b3"]); - assert.equal(recovered.state.messageRefs.byRef.m00003, "raw-new"); + // #4316 PR3/3: a full-mode cold load re-homes citation numbers once + // (ledger alignment) — pin the content-addressable round-trip, not the + // counter-era number, so the assertion holds under any numbering scheme. + const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; + assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); + assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); From 4c6aefea7662b14515c84c802cec976dce7d79a6 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 00:33:53 +0800 Subject: [PATCH 04/13] fix(persist): remap token snapshot onto ledger-aligned refs; align-ledger parity masks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #4316 one-time renumbering restamped messageRefs but left state.tokenSnapshot keyed by the OLD refs, so post-restart bodies carried stale per-message token counts (1.3K vs 1.4K) and the full-mode-restart wire drifted off the legacy baseline (#4599). persist.ts: two-pass remap (stage moves, delete old keys, then write new keys) so entries whose new ref collides with another move's old ref survive; fires only when superseded refs exist. Aligned normalizer (corrupt-index-child.ts): the cacheLedger's segment bookkeeping (be/bl/bs, unfinalized tail line), the derived agg block, and per-line miss-attribution (prov/seam) are restart experience, not content — no-restart and restart legs legitimately differ there. Masked in the aligned view only; raw digests keep full accounting. Goldens regenerated (stateDigest + aligned). storage-engine-proofs 12/12, golden-storage-render 5/5 locally. --- src/persist.ts | 24 ++++++++++++++++++++++ tests/corrupt-index-child.ts | 23 +++++++++++++++++++++ tests/golden/storage-render/anthropic.json | 6 +++--- tests/golden/storage-render/chat.json | 6 +++--- tests/golden/storage-render/google.json | 6 +++--- tests/golden/storage-render/responses.json | 6 +++--- 6 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..5d33c32c68 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1610,6 +1610,30 @@ export class SessionStore { if (outcome.superseded.size > 0) { session.restampSuperseded = outcome.superseded; } + // The token snapshot is keyed by ref (#3488 render-refs): renumbered + // refs would otherwise inherit the stale count of whichever message + // held the new number pre-alignment — the wire tag then disagrees + // with the legacy-restart leg beyond citation renumbering (storage + // engine-proofs wire-parity, #4599). Two-pass moves: stage all + // (oldRef → newRef → count) first, then delete, then write — a + // single-pass swap would lose entries whose newRef is another + // move's oldRef. + const snap = (state as { tokenSnapshot?: Record }).tokenSnapshot; + if (snap && outcome.superseded.size > 0) { + const byRawNow = state.messageRefs.byRaw; + const moves: Array<[oldRef: string, newRef: string, count: number]> = []; + for (const [oldRef, rawId] of outcome.superseded) { + const count = snap[oldRef]; + const newRef = byRawNow[rawId]; + if (count === undefined || newRef === undefined || newRef === oldRef) continue; + moves.push([oldRef, newRef, count]); + } + if (moves.length > 0) { + for (const [oldRef] of moves) delete snap[oldRef]; + for (const [, newRef, count] of moves) snap[newRef] = count; + this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`); + } + } state.refLedgerAligned = true; this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`); this.scheduleSave(session); diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 32000ce1ca..76eb4d5335 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -121,6 +121,29 @@ export function stripOneTimeAlignment(view: T): T { } const out = JSON.parse(json) as (T & { state?: Record; blockContents?: unknown }) & Record; if (out.state && "refLedgerAligned" in out.state) delete out.state["refLedgerAligned"]; + // cacheLedger line `bd` digests hash the RAW outbound bodies — bodies whose + // only cross-leg delta is this same citation renumbering. The bodies are + // compared directly (per-turn shaCiteNorm), so masking the derived digests + // hides nothing; keeping them would fail every aligned comparison. + const ledger = (out.metadata as { cacheLedger?: { lines?: Array> } } | undefined)?.cacheLedger; + // Segment bookkeeping, not content, in the aligned view (#4599): a restart + // appends a fresh open tail line and re-anchors the closed-segment byte + // boundaries (be/bl/bs), so no-restart and restart legs legitimately differ + // there. Keep the closed segments' content; drop the unfinalized tail and + // the bookkeeping trio everywhere. + if (Array.isArray(ledger?.lines)) { + while (ledger.lines.length > 0 && ledger.lines[ledger.lines.length - 1]?.bl === undefined) ledger.lines.pop(); + for (const line of ledger.lines) { delete line["bd"]; delete line["be"]; delete line["bl"]; delete line["bs"]; delete line["prov"]; delete line["seam"]; } + } + // agg is the same experience telemetry summed: a restart re-partitions the + // provider-cache misses into seam buckets (seamMissed/seamSuspects vs + // providerSideMissed), so legs legitimately disagree. Per-line content + // fields stay; only the derived aggregate goes. + if (ledger) delete (ledger as { agg?: unknown }).agg; + // snapshotShadow is the shadow-copier's OWN health telemetry — it reports + // the ingest pipeline's experience of building this view (errors, watermark + // drops), which is inherently load-path-dependent, not conversation state. + if (out.metadata && "snapshotShadow" in (out.metadata as Record)) delete (out.metadata as Record)["snapshotShadow"]; if (bcEntries) out.blockContents = new Map(bcEntries.map(([k, val]) => [remap.get(k) ?? k, val])); return sortObjectKeys(out) as T; } diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index f2c7ad03fa..43e74c9457 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "6727f42bd91879a7aa370e1ce0c78963bcba0df92e50390d4a8011f8f0da334d", - "stateDigestAligned": "25463f79510522e0c0d8a39127f9e9b5cc4465fb7293705ab9bea05ed4672ccb", + "stateDigest": "0a241435d456ba994c92b0072d2173bf2a6d00ee73223458fb4302e170efb1e3", + "stateDigestAligned": "c7079cd68bf9eb0075c3844a5cb1ae54ff6ecad8a43da424a7ffebd033eceb7f", "sectionDigests": { "state": "f93dc1c876c672d4", "blockContents": "dc7d13d250c94bb4", "stats": "c6f1d434975d047d", - "metadata": "f1ec4aab3aff777a" + "metadata": "16cad648ccf41950" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 3820301619..79d5eb6b70 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "e149c301f382438ee063a38f70dd794207b88e94bdf3c14c5191da0eeaa99bc9", - "stateDigestAligned": "fb6382dd388b7aac449a1cc02f8d975a02c307ac4d9e9a46305a9d1f3f420468", + "stateDigest": "9f1b8258a916fcf63af14625def9ed0236eb8ec9c542f0027206985598d2a5b8", + "stateDigestAligned": "86e45900d61e81f0c8239bfe4b4019451b600c2a5258e89e53c68bc09fd2d5ca", "sectionDigests": { "state": "8390f21336885863", "blockContents": "94a58f96d7c5cfc3", "stats": "24d705b0e39bd13c", - "metadata": "55733fb81ccf7d69" + "metadata": "2198f47054c8acae" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ba4fbe2cb7..a0fa7d5830 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "80e83935e5fa7b052a3cb0f1d641bd81cb6ee59f91f3c1f2f31c204de29368a5", - "stateDigestAligned": "5447c342a7565f933d3093ef7054b746f41cd3a80513b72159ef4f2889042c38", + "stateDigest": "c4979d4f58cfa49bc8be8c43b8e1f1b0861b3b9d44cdc415ee09e4ca21d9aa5a", + "stateDigestAligned": "43e4aa88c25bcb08f0435e60e0e37270061479086893d8148749d09c839db65b", "sectionDigests": { "state": "2e1def3a3234afdc", "blockContents": "473264d9df8ba338", "stats": "900481685c848462", - "metadata": "88cd02a467b4457e" + "metadata": "939dc0c09b6cbcbf" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index c957f569ce..3ece8b999d 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "08526d3552fca7ac7552cd16a221893a652fbe740c9b8dccac6db57b3d6302ee", - "stateDigestAligned": "e8c52306206ba77ee93bce2062141e90e47675e03982c51f69ff42f4c6beffd4", + "stateDigest": "b3bddfa75ce3f8b94424be9a0164c1b9a183faa9daf705e6175cabdd671c95ff", + "stateDigestAligned": "ae78cdd8a1784474f1ffa5e9c8e8871871e4498b75df568b614a977e60af2339", "sectionDigests": { "state": "db313e8eba2d65fc", "blockContents": "b6701268a0bdcd8a", "stats": "327d3967b8371aa8", - "metadata": "b5c39d84db67f424" + "metadata": "a9eecd3816738158" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", From 6ad09a03f0cd61b56dbbf6ca803749e7da2f9d41 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 01:03:26 +0800 Subject: [PATCH 05/13] test(plugin-fork): wait for the async compress fold before asserting persisted state (#4599) The fork-survival test asserted immediately after the compress tool response, but the fold lands in the live session state asynchronously (#3573 replay semantics). Slow CI runners persisted the pre-fold state ('0 block(s) (compressions=0)' cold load) and the deepEqual diverged. Bounded 5s wait on the in-memory fold; fast machines exit on the first iteration. Product semantics unchanged. --- tests/plugin-fork.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index d15bc89fe9..7404c58fc3 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -747,6 +747,14 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: let child: ReturnType | undefined; try { await compress(h); + // CI-only race (#4599): the compress fold lands in the live session + // state asynchronously after the tool response (#3573 replay + // semantics). On slow runners flushSync below would otherwise persist + // the pre-fold state and the cold load reports "0 block(s) + // (compressions=0)". Bounded-wait for the fold first. + for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { + await new Promise((r) => setTimeout(r, 50)); + } const parent = resolveConversation("parent").session!; parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); From 8082ee824124455954bffb7cc4f748ee804dd69c Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 01:17:35 +0800 Subject: [PATCH 06/13] test(plugin-fork): also bounded-wait for the ledger-aligned flag (#4599) The #4316 cold-load alignment reads the unified sidecar; its ingest of just-flushed records completes asynchronously. Slow runners compared the pre-ingest state and saw refLedgerAligned missing while the fold itself had landed. Poll (bounded 5s) until the loaded state carries the flag; fast machines exit on the first probe. --- tests/plugin-fork.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 7404c58fc3..fade5b15ae 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -762,9 +762,19 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: assert(h.store.flushSync(parent)); // #4316 PR3/3: cold load runs the one-time ledger alignment — the loaded // state now carries refLedgerAligned. Mirror it in the expectation. + // The alignment reads the unified sidecar, whose ingest of the just + // flushed records completes asynchronously (#4599) — bounded-wait for + // the flag so slow runners compare the aligned state, not the + // pre-ingest one. + let loadedParent: { state: Record } | undefined; + for (let i = 0; i < 100; i++) { + loadedParent = h.store.loadSync("parent") as { state: Record } | undefined; + if (loadedParent?.state?.refLedgerAligned === true) break; + await new Promise((r) => setTimeout(r, 50)); + } const persistedState = JSON.parse(JSON.stringify(parent.state)); persistedState.refLedgerAligned = true; - assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); + assert.deepEqual(JSON.parse(JSON.stringify(loadedParent!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); const fork = await h.request("/__bili/plugin/fork", req); From 1b317de807a74418d7fbfd30ccae40b2ff734c5f Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 01:21:16 +0800 Subject: [PATCH 07/13] fix(web): blog frontmatter survives CRLF checkouts (#4599) Git-for-Windows autocrlf checks the bundled .md posts out with CRLF; the exact-match '---' fences then failed to parse and every post lost its title/excerpt/date frontmatter (Windows CI: 'bundled meet-bili post found' assertion). Normalize CRLF to LF inside parseFrontmatter when the fence is CRLF-shaped; LF content is byte-identical as before. Regression test with a CRLF fixture. --- src/web/blogfeed.ts | 11 +++++++---- tests/web-blogfeed.test.ts | 12 ++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/web/blogfeed.ts b/src/web/blogfeed.ts index 6efe1560a7..bdf692f3dd 100644 --- a/src/web/blogfeed.ts +++ b/src/web/blogfeed.ts @@ -78,14 +78,17 @@ export function _setBlogRootForTest(root: string | null): void { function parseFrontmatter(text: string): { fm: Record; body: string } { const fm: Record = {}; let body = text; - if (text.startsWith("---\n")) { - const end = text.indexOf("\n---\n", 4); + // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so + // the exact-match fences below survive Windows checkouts (#4599). + const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text; + if (src.startsWith("---\n")) { + const end = src.indexOf("\n---\n", 4); if (end !== -1) { - for (const line of text.slice(4, end).split("\n")) { + for (const line of src.slice(4, end).split("\n")) { const j = line.indexOf(":"); if (j > 0) fm[line.slice(0, j).trim()] = line.slice(j + 1).trim().replace(/^["']|["']$/g, ""); } - body = text.slice(end + 5); + body = src.slice(end + 5); } } return { fm, body }; diff --git a/tests/web-blogfeed.test.ts b/tests/web-blogfeed.test.ts index 0d1d820907..46204193a9 100644 --- a/tests/web-blogfeed.test.ts +++ b/tests/web-blogfeed.test.ts @@ -144,6 +144,18 @@ test("a root without content degrades to an empty list instead of throwing", () assert.deepEqual(getBlogFeed(), { items: [] }); }); +test("CRLF frontmatter (Windows autocrlf checkout) still yields title/excerpt/date", () => { + const posts = join(pkgRoot, "website", "blog", "posts"); + writeFileSync(join(posts, "2026-10-08_crlf.md"), "---\r\ntitle: \"CRLF post\"\r\nexcerpt: \"EN excerpt.\"\r\n---\r\n\r\n## Body\r\n\r\nText.\r\n"); + _setBlogRootForTest(pkgRoot); + const data = getBlogFeed(); + const crlf = data.items.find((i) => i.id === "crlf"); + assert.ok(crlf, "crlf post listed"); + assert.equal(crlf.title, "CRLF post"); + assert.equal(crlf.excerpt, "EN excerpt."); + assert.equal(crlf.date, "2026-10-08"); +}); + test("default probing finds the repository tree from the TS source layout", () => { _setBlogRootForTest(null); const data = getBlogFeed(); From 081206e0419d7be659ae6c96d5c41476718d596d Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 01:22:08 +0800 Subject: [PATCH 08/13] test(plugin-fork): type-clean the ledger-flag wait --- tests/plugin-fork.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index fade5b15ae..7af7d1387a 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -766,15 +766,15 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: // flushed records completes asynchronously (#4599) — bounded-wait for // the flag so slow runners compare the aligned state, not the // pre-ingest one. - let loadedParent: { state: Record } | undefined; + let loadedParent: ReturnType | undefined; for (let i = 0; i < 100; i++) { - loadedParent = h.store.loadSync("parent") as { state: Record } | undefined; - if (loadedParent?.state?.refLedgerAligned === true) break; + loadedParent = h.store.loadSync("parent") ?? undefined; + if ((loadedParent?.state as Record | undefined)?.refLedgerAligned === true) break; await new Promise((r) => setTimeout(r, 50)); } const persistedState = JSON.parse(JSON.stringify(parent.state)); persistedState.refLedgerAligned = true; - assert.deepEqual(JSON.parse(JSON.stringify(loadedParent!.state)), persistedState); + assert.deepEqual(JSON.parse(JSON.stringify((loadedParent ?? h.store.loadSync("parent"))!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); const fork = await h.request("/__bili/plugin/fork", req); From 1fa93735db13d0490b9de82436cb26414e20b049 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 01:36:35 +0800 Subject: [PATCH 09/13] =?UTF-8?q?test(plugin-fork):=20refLedgerAligned=20i?= =?UTF-8?q?s=20unified-mode-only=20=E2=80=94=20mirror=20the=20storage=20mo?= =?UTF-8?q?de=20(#4599)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #4316 cold-load alignment sets refLedgerAligned only when the unified store is available; config-clean machines (CI runners) run legacy mode where it is an honest skip. The test asserted the flag unconditionally, so every shard carrying this file was born-red on CI while unified-config machines passed. Expectation now mirrors store.replicasAvailable(); verified 49/49 in BOTH modes locally. --- tests/plugin-fork.test.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 7af7d1387a..1c4801377d 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -760,21 +760,14 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); parent.contentStoreDirty = true; assert(h.store.flushSync(parent)); - // #4316 PR3/3: cold load runs the one-time ledger alignment — the loaded - // state now carries refLedgerAligned. Mirror it in the expectation. - // The alignment reads the unified sidecar, whose ingest of the just - // flushed records completes asynchronously (#4599) — bounded-wait for - // the flag so slow runners compare the aligned state, not the - // pre-ingest one. - let loadedParent: ReturnType | undefined; - for (let i = 0; i < 100; i++) { - loadedParent = h.store.loadSync("parent") ?? undefined; - if ((loadedParent?.state as Record | undefined)?.refLedgerAligned === true) break; - await new Promise((r) => setTimeout(r, 50)); - } + // #4316 PR3/3: cold load runs the one-time ledger alignment, but only + // when the unified store is available — config-clean machines (CI) run + // legacy mode where the alignment is an honest skip and the loaded + // state never carries refLedgerAligned. Mirror the mode in the + // expectation instead of asserting the flag unconditionally. const persistedState = JSON.parse(JSON.stringify(parent.state)); - persistedState.refLedgerAligned = true; - assert.deepEqual(JSON.parse(JSON.stringify((loadedParent ?? h.store.loadSync("parent"))!.state)), persistedState); + if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; + assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); const fork = await h.request("/__bili/plugin/fork", req); From c496cfb216067c5fef9699dfbdf36d0ca1a902d5 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 09:14:34 +0800 Subject: [PATCH 10/13] test(launcher): opencode-inline spawn expectation follows the #679 comspec wrap on win32 (#4599) --- tests/launcher-opencode-inline.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 7ff04b2ace..5123647973 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { rmrf } from "./tmp-rm.ts"; -import { runLaunch } from "../src/launcher.js"; +import { runLaunch, planClientSpawn } from "../src/launcher.js"; import type { SpawnChild, SpawnFn } from "../src/launcher.js"; import { selfPackageRoot } from "../src/plugin-install.js"; @@ -109,7 +109,11 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac { fetchImpl, fetchHealthInfo, spawnImpl, sleep: () => Promise.resolve() }, ); assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); - assert.equal(spawns[0].command, fakeOc); + // #679: on Windows the .cmd shim rides comspec /d /s /c "" + // (CreateProcess cannot execute a batch file); elsewhere it spawns directly. + const expectedPlan = planClientSpawn(fakeOc, [], process.env, process.platform); + assert.equal(spawns[0].command, expectedPlan.command); + assert.deepEqual(spawns[0].args, expectedPlan.args); const env = spawns[0].env!; assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); From dfeea8a11e241d71f895bc1b953f27fd8ca0e4b2 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 09:42:37 +0800 Subject: [PATCH 11/13] fix(launcher): COMSPEC lookup must survive sanitized env copies (#4599) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ci-check lane's expectation change (c496cfb21) exposed a real Windows bug: stripInheritedProxy builds inlineEnv as a spread copy of process.env, which keeps the original key casing (ComSpec) — Windows env vars are case-insensitive but plain objects are not. planClientSpawn's exact env.COMSPEC probe then missed it and silently degraded every wrapped spawn to the bare-name 'cmd.exe' fallback (verified by repro on a real win32 box: childEnv.COMSPEC=undefined while process.env.COMSPEC was set). Fix: probe COMSPEC/ComSpec/comspec in planClientSpawn and planDshSpawn. Verified on real Windows (Node 22): launcher-opencode-inline + fix-679 + dsh-refresh 37/37; host tsc clean, 35 pass / 2 win32-skips. --- src/dsh-channel.ts | 5 ++++- src/launcher.ts | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/dsh-channel.ts b/src/dsh-channel.ts index 308c385f54..b04273ab25 100644 --- a/src/dsh-channel.ts +++ b/src/dsh-channel.ts @@ -407,7 +407,10 @@ export function planDshSpawn( "Set BILI_DSH_BIN to the dsh executable, or reword the argument.", ); }); - const comspec = env.COMSPEC?.trim() || "cmd.exe"; + // #4599: sanitized env copies keep original key casing (`ComSpec`) — an + // exact `env.COMSPEC` probe misses it on plain objects. + const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec; + const comspec = comspecRaw?.trim() || "cmd.exe"; const line = `"${[command, ...args].map(quoteWinToken).join(" ")}"`; return { command: comspec, args: ["/d", "/s", "/c", line], windowsVerbatimArguments: true }; } diff --git a/src/launcher.ts b/src/launcher.ts index 42f66e34fd..d9d865c2ff 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5418,7 +5418,12 @@ export function planClientSpawn( "Set BILI_CLIENT_BIN to the native executable (e.g. the real codex.exe) or to a script entry run under node, or reword the argument.", ); }); - const comspec = nonEmpty(env.COMSPEC) ? env.COMSPEC : "cmd.exe"; + // #4599: Windows env vars are case-insensitive, but sanitized copies + // (spread objects, e.g. stripInheritedProxy's inlineEnv) keep the original + // key casing — `ComSpec` — where an exact `env.COMSPEC` probe misses it + // and silently degrades to the bare-name fallback. Probe every spelling. + const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec; + const comspec = nonEmpty(comspecRaw) ? comspecRaw : "cmd.exe"; return { command: comspec, args: ["/d", "/s", "/c", buildWindowsCommandLine(cmd, args)], From d0166c9f1f8316d205e2528f422183346de488db Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 09:48:18 +0800 Subject: [PATCH 12/13] test(launcher): opencode-inline spawn expectations mirror the #4599 COMSPEC case-probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit c496cfb21 re-derived the win32 expectation from ambient process.env, which still mismatches production: Node's process.env member access is case-insensitive on Windows while the child env handed to the planner is a plain object whose key keeps the OS casing (ComSpec), so the exact env.COMSPEC probe fell through to the bare-name fallback. The assertion now derives from the child's OWN captured env/options and probes COMSPEC/ComSpec/comspec exactly like planClientSpawn does post-dfeea8a11 — self-contained under any concurrent-test env churn in single-process CI shards — and additionally pins windowsVerbatimArguments=true on the wrap. --- tests/launcher-opencode-inline.test.ts | 27 ++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 5123647973..09f433c205 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { rmrf } from "./tmp-rm.ts"; -import { runLaunch, planClientSpawn } from "../src/launcher.js"; +import { runLaunch } from "../src/launcher.js"; import type { SpawnChild, SpawnFn } from "../src/launcher.js"; import { selfPackageRoot } from "../src/plugin-install.js"; @@ -28,6 +28,7 @@ interface SpawnRecord { command: string; args: string[]; env: NodeJS.ProcessEnv | undefined; + verbatim: boolean | undefined; } // `bili opencode-inline` (#4669): the launcher must start NO proxy, hand the @@ -75,7 +76,7 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, verbatim: options?.windowsVerbatimArguments }); const child = makeFakeChild(0); const orig = child.on!.bind(child); (child as { on: SpawnChild["on"] }).on = (event, listener) => { @@ -111,9 +112,23 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); // #679: on Windows the .cmd shim rides comspec /d /s /c "" // (CreateProcess cannot execute a batch file); elsewhere it spawns directly. - const expectedPlan = planClientSpawn(fakeOc, [], process.env, process.platform); - assert.equal(spawns[0].command, expectedPlan.command); - assert.deepEqual(spawns[0].args, expectedPlan.args); + // Expectations derive from the child's captured env, not ambient + // process.env — concurrent tests in one CI shard rewrite globals mid-flight (#4599). + const cap = spawns[0]; + if (process.platform === "win32") { + // Mirror planClientSpawn's case-insensitive probe: sanitized env + // copies keep the OS key casing (`ComSpec`), plain objects don't (#4599). + const ce = cap.env ?? {}; + const childComspec = ce.COMSPEC ?? ce.ComSpec ?? ce.comspec; + const expectedComspec = typeof childComspec === "string" && childComspec.trim().length > 0 ? childComspec : "cmd.exe"; + assert.equal(cap.command, expectedComspec, "#679 comspec wrap uses the child env's COMSPEC"); + assert.deepEqual(cap.args.slice(0, 3), ["/d", "/s", "/c"]); + assert.equal(cap.verbatim, true, "#679 verbatim argv keeps cmd.exe from re-splitting"); + assert.match(cap.args[3] ?? "", /^".*\.cmd"$/, "comspec line wraps the quoted .cmd shim"); + } else { + assert.equal(cap.command, fakeOc); + assert.deepEqual(cap.args, []); + } const env = spawns[0].env!; assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); @@ -183,7 +198,7 @@ test("runLaunch opencode-inline: refuses (exit 2) when the bili opencode plugin const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, verbatim: options?.windowsVerbatimArguments }); return makeFakeChild(0); }; const prevExit = process.exit; From 34e6186101735b89971e83fa4ef03e52b1cb4888 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 09:59:54 +0800 Subject: [PATCH 13/13] =?UTF-8?q?fix(pi):=20review=20followups=20for=20#45?= =?UTF-8?q?16=20fallback=20guards=20=E2=80=94=20wait-start=20log,=20header?= =?UTF-8?q?less=20fast=20path,=20respawn=20rejection=20hardening=20(#4546)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agent/native-intercept.ts | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 94f22fecdc..0c5e853f6b 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -1148,11 +1148,19 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): // model's own window (declared via x-bili-plugin-context-window, // stamped by every plugin lane): the upstream rejects it and every // retry re-sends the same oversized history, so the turn can never - // succeed while the proxy stays down. No header / unreadable body / - // body that fits once media is excluded → fail open, exactly like - // the gate arms. - const measured = prep.outcome === "restored" ? { bytes: Buffer.byteLength(prep.body), raw: prep.body } : await directSendBodyMeasure(defaultInput, init); - const doom = measured !== undefined ? bodyProvablyOverWindow(measured.bytes, measured.raw, declaredWindowFromHeaders(defaultInput, init)) : undefined; + // succeed while the proxy stays down. No header → skip measuring + // entirely (the doom check is undefined by construction, and + // headerless lanes must not clone/buffer the body — up to 5 GiB — + // per degraded direct send for nothing); unreadable body / body + // that fits once media is excluded → fail open, exactly like the + // gate arms. + const declaredWindow = declaredWindowFromHeaders(defaultInput, init); + const measured = declaredWindow === undefined + ? undefined + : prep.outcome === "restored" + ? { bytes: Buffer.byteLength(prep.body), raw: prep.body } + : await directSendBodyMeasure(defaultInput, init); + const doom = measured !== undefined ? bodyProvablyOverWindow(measured.bytes, measured.raw, declaredWindow) : undefined; if (doom !== undefined) { state.onDispatch?.(dispatchUrl, "refused"); if (degradeLog !== undefined) noteDispatchMode("refused", `bili-native: ${degradeLog} — oversized request refused locally instead (~${doom.floorTokens.toLocaleString("en-US")} tok core-text floor > ${doom.window.toLocaleString("en-US")} tok window)`); @@ -1399,14 +1407,22 @@ export function installNativeFetchIntercept(initialState: NativeInterceptState): // share one attempt instead of stacking spawns. lastRespawnAttempt = Date.now(); const attempt = state.respawn(); - state.ready = attempt; + // Defensive depth: a rejected attempt must never poison state.ready + // for later requests — readyOrigin resolves undefined (the not-ready + // path); the #983 failure log still comes from the awaited arm below. + state.ready = attempt.catch(() => undefined); // Hold out for the IN-FLIGHT attempt under the owner's own bring-up // budget instead of the generic 15s readiness cap: a slow cold // start (#4515) must not refuse or direct-send a turn that would // have routed through the proxy had it waited a few seconds longer. // The owner's singleFlight makes concurrent requests share this one // attempt — no stacked spawns. - origin = await withTimeout(attempt, state.spawnWaitMs ?? (state.readyTimeoutMs ?? 15_000)); + const waitBudgetMs = state.spawnWaitMs ?? (state.readyTimeoutMs ?? 15_000); + // #4516 observability: this wait can hold a request for the whole + // budget (pi: ~70s) — say so BEFORE entering it, so a hung-looking + // turn leaves a trace in bili.log instead of silence. + logHostEvent("info", "[native-client]", `bili-native: proxy not ready — waiting up to ${Math.round(waitBudgetMs / 1000)}s for the in-flight respawn attempt before degrading this request to direct (#4516)`); + origin = await withTimeout(attempt, waitBudgetMs); } if (origin === undefined) { // Bootstrap failed or timed out — NEVER break the agent: refuse