Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 85 additions & 1 deletion src/external-summary-credentials.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,29 @@ import { closeSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readF
import { dirname } from "node:path";
import { randomUUID } from "node:crypto";
import { configFile } from "./paths.js";
import { validSummaryCredentialName } from "./external-summary-settings.js";
import { loadNamedProviders, loadRoutes, safeReadJson, type NamedProviderRecipe } from "./config.js";
import { expandExternalSummaryChain, parseExternalSummaryChain, validSummaryCredentialName } from "./external-summary-settings.js";

function validKey(value: unknown): value is string {
return typeof value === "string" && value.length > 0 && value.length <= 8192 && !/[\x00-\x20\x7f]/.test(value);
}

/** #3488: resolve() has three distinct failure shapes (reference unset, value
* unusable, store unreadable) plus success; configured() collapsed all of
* them to `false`, so the UI could not tell an unset env var from a value
* full of whitespace or a corrupt private store. */
export type SummaryCredentialState = "configured" | "missing" | "invalid_value" | "store_unreadable" | "invalid_reference";

export function describeCredentialState(reference: string, state: SummaryCredentialState): string {
switch (state) {
case "configured": return `${reference} resolves to a usable credential`;
case "missing": return reference.startsWith("env:") ? `environment variable ${reference.slice(4)} is not set` : `credential "${reference.slice(7)}" is not in the private store`;
case "invalid_value": return `environment variable ${reference.slice(4)} holds an unusable value (empty, whitespace/control characters, or over 8192 chars)`;
case "store_unreadable": return "the private credential store is unreadable (permissions or contents)";
case "invalid_reference": return `${reference} is not a valid credential reference`;
}
}

/** Separate from configFile(): normal config GET/raw exports never read these bytes. */
export class SummaryCredentialStore {
constructor(private readonly path: string = `${configFile()}.summary-credentials.json`) {}
Expand Down Expand Up @@ -43,6 +60,20 @@ export class SummaryCredentialStore {
try { return this.resolve(reference, env) !== undefined; } catch { return false; }
}

status(reference: string, env: NodeJS.ProcessEnv = process.env): SummaryCredentialState {
if (/^env:[A-Za-z_][A-Za-z0-9_]*$/.test(reference)) {
const value = env[reference.slice(4)];
return value === undefined ? "missing" : validKey(value) ? "configured" : "invalid_value";
}
if (reference.startsWith("secret:") && validSummaryCredentialName(reference.slice(7))) {
let values: Record<string, string>;
try { values = this.read(); } catch { return "store_unreadable"; }
// Stored values are validated on write, so a present entry is always usable.
return values[reference.slice(7)] === undefined ? "missing" : "configured";
}
return "invalid_reference";
}

set(name: string, key: string | null): void {
if (!validSummaryCredentialName(name) || (key !== null && !validKey(key))) throw new Error("Invalid external summary credential");
mkdirSync(dirname(this.path), { recursive: true, mode: 0o700 });
Expand Down Expand Up @@ -71,3 +102,56 @@ export class SummaryCredentialStore {
}
}
}

/** #3488: readable problems for ONE raw chain block (file form). Unparseable
* blocks and unresolvable references both return [] — those surfaces are
* owned elsewhere (config validation / web save 400 / the request-path
* one-shot warn), and boot cannot see agent-registered recipes (#2336), so
* reporting a reference problem here would false-alarm on every start for
* chains that resolve fine once the plugin host registers its layer. */
export function auditExternalSummaryCredentials(block: unknown, recipes: Record<string, NamedProviderRecipe>, env: NodeJS.ProcessEnv = process.env): string[] {
let chain;
try { chain = parseExternalSummaryChain(block); } catch { return []; }
if (!chain.enabled) return [];
let targets;
try { targets = expandExternalSummaryChain(chain, recipes).targets; }
catch { return []; }
const store = new SummaryCredentialStore();
const problems: string[] = [];
for (const target of targets) {
const ref = target.credentialRef ?? (target.apiKey === undefined ? "" : "(inline)");
if (ref === "") problems.push(`target ${target.name}: recipe carries no apiKeyEnv/credentialRef and no inline key`);
else if (ref !== "(inline)") {
const state = store.status(ref, env);
if (state !== "configured") problems.push(`target ${target.name}: ${describeCredentialState(ref, state)}`);
}
}
return problems;
}

/** #3488: boot-time scan of every EFFECTIVE enabled external-summary chain.
* Walks the same cascade mergeCompress uses (global → route → model, whole
* chain replace) so shadowed blocks are not reported, and identity-dedupes
* because one file object can be effective for many slots. Returns
* human-readable problem lines (empty when everything resolves). */
export function auditStartupSummaryCredentials(env: NodeJS.ProcessEnv = process.env): string[] {
const file = safeReadJson(configFile());
const compress = file && typeof file === "object" && !Array.isArray(file) ? (file as { compress?: unknown }).compress : undefined;
const globalBlock = compress && typeof compress === "object" && !Array.isArray(compress) ? (compress as { externalSummary?: unknown }).externalSummary : undefined;
const routes = loadRoutes(env);
const recipes = loadNamedProviders(env);
const seen = new Set<unknown>();
const problems: string[] = [];
const audit = (block: unknown): void => {
if (block === undefined || seen.has(block)) return;
seen.add(block);
for (const problem of auditExternalSummaryCredentials(block, recipes, env)) problems.push(problem);
};
audit(globalBlock);
for (const route of Object.values(routes)) {
const routeBlock = route.compress?.externalSummary;
audit(routeBlock ?? globalBlock);
for (const model of Object.values(route.models ?? {})) audit(model.compress?.externalSummary ?? routeBlock ?? globalBlock);
}
return problems;
}
129 changes: 115 additions & 14 deletions src/external-summary-http.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,84 @@
import type { SummaryCandidate, SummaryWork } from "./external-summary.js";
import { defaultCountTokens } from "acp-kernel";
import { fetchWithTimeout } from "./fetch-util.js";
import { classifyUpstreamFailure, isFailFastUpstreamKind } from "./upstream-fail.js";
import { fetchWithTimeout, isTransientUpstreamError, sleep } from "./fetch-util.js";
import { extractSummaryFromSse, extractSummaryText, summaryPayload, type PreflightProtocol } from "./preflight.js";
import { log as loggerLog } from "./logger.js";
import { maskUrlForLog, maskUrlsInText } from "./log-mask.js";
import { normalizeSseLineEndings } from "./sse-util.js";
import { proxyDispatcher } from "./upstream-proxy.js";
import { PACKAGE_NAME, VERSION } from "./version.js";

/** #3488: bounded retry budget for one candidate call. Total dispatches
* (initial + retries). Retries fire ONLY on transient verdicts and must fit
* the caller's per-attempt deadline (the executor aborts the attempt signal
* at targetTimeoutMs), so the wall-clock bound is unchanged from today's
* single dispatch — a retry can only recover a fast-failing transient, never
* lengthen a pass. */
const SUMMARY_MAX_ATTEMPTS = 3;
const SUMMARY_RETRY_BASE_DELAY_MS = 1_000;
const SUMMARY_RETRY_MAX_DELAY_MS = 8_000;
/** A hostile Retry-After header must not eat the whole attempt budget. */
const SUMMARY_RETRY_AFTER_CAP_MS = 15_000;
/** Failure bodies are logged truncated: enough to distinguish auth vs
* rate-limit vs risk-control text without dumping conversation content. */
const SUMMARY_ERROR_BODY_LOG_CHARS = 1_024;
/** How much of a failure body to PULL at all — a hostile 5xx must not be
* buffered whole just to log its first kilobyte. */
const SUMMARY_ERROR_BODY_READ_CHARS = 8_192;

let summaryRetryDelayOverrideMs: number | null = null;
const scheduledSummaryRetryDelaysMs: number[] = [];
/** Test seam (#3488): override the actual sleep duration and record each
* scheduled delay so tests can assert backoff/Retry-After math without
* wall-clock waits. null = production behavior. */
export function _setSummaryRetryDelayForTest(overrideMs: number | null): void {
summaryRetryDelayOverrideMs = overrideMs;
scheduledSummaryRetryDelaysMs.length = 0;
}
export function _summaryRetryDelaysForTest(): readonly number[] {
return scheduledSummaryRetryDelaysMs;
}

function backoffDelayMs(attempt: number): number {
return Math.min(SUMMARY_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), SUMMARY_RETRY_MAX_DELAY_MS);
}

function retryAfterDelayMs(header: string | null, fallbackMs: number): number {
if (header !== null) {
const seconds = Number(header);
if (Number.isFinite(seconds) && seconds >= 0) return Math.min(Math.ceil(seconds * 1000), SUMMARY_RETRY_AFTER_CAP_MS);
const dateMs = Date.parse(header);
if (!Number.isNaN(dateMs)) return Math.min(Math.max(dateMs - Date.now(), 0), SUMMARY_RETRY_AFTER_CAP_MS);
}
return fallbackMs;
}

async function waitBeforeRetry(ms: number, signal: AbortSignal): Promise<void> {
// Recording is a test seam: in production (override null) the array must
// stay empty or every transient retry leaks entries into it forever.
if (summaryRetryDelayOverrideMs !== null) scheduledSummaryRetryDelaysMs.push(ms);
await sleep(summaryRetryDelayOverrideMs ?? ms, signal);
}

async function readErrorSnippet(response: Response): Promise<string> {
try {
const reader = response.body?.getReader();
if (!reader) return "";
const decoder = new TextDecoder("utf-8");
let text = "";
for (;;) {
const { done, value } = await reader.read();
if (done) break;
text += decoder.decode(value, { stream: true });
if (text.length >= SUMMARY_ERROR_BODY_READ_CHARS) { await reader.cancel().catch(() => { }); break; }
}
const trimmed = text.trim().replace(/\s+/g, " ");
if (!trimmed) return "";
return trimmed.length > SUMMARY_ERROR_BODY_LOG_CHARS ? `${trimmed.slice(0, SUMMARY_ERROR_BODY_LOG_CHARS)}…` : trimmed;
} catch { return ""; }
}

/** Internal resolved request plan, not a user-config schema or credential store. */
export interface SummaryHttpTarget {
readonly protocol: PreflightProtocol;
Expand Down Expand Up @@ -160,22 +233,50 @@ export function createSummaryHttpCandidate(target: SummaryHttpTarget, maxRespons
}
}
// Reuse the existing codecs without carrying the main request's auth or
// model. Each candidate dispatches exactly once. (The Go-specific
// model. Bounded retry (#3488): transient verdicts (429/5xx/risk-control
// body markers, fail-fast pre-response network deaths) re-dispatch up to
// SUMMARY_MAX_ATTEMPTS times total, honoring Retry-After within the cap.
// Every failed attempt lands in the local log with status + truncated
// body — the executor maps errors to outcome enums, so without this the
// operator sees nothing but `attempts=[t0:error]`. (The Go-specific
// x-opencode-session / user-agent above are derived from bili's own
// session id, not copied from the client's inbound headers — #2760.)
const { response, clearTimer } = await fetchWithTimeout(plan.url, {
method: "POST", headers: plan.headers, body: JSON.stringify(payload), redirect: "manual", dispatcher: proxyDispatcher(plan.proxyUrl),
}, undefined, signal);
try {
if (!response.ok) {
try { await response.body?.cancel(); } catch { /* transport may already be closed */ }
throw new Error("External summary upstream rejected the request");
}
const result = parseSummary(plan.protocol, await readResponse(response, maxResponseBytes));
const host = maskUrlForLog(plan.url);
let attempt = 0;
for (;;) {
attempt += 1;
signal.throwIfAborted();
return result;
} finally {
clearTimer();
let dispatched;
try {
dispatched = await fetchWithTimeout(plan.url, {
method: "POST", headers: plan.headers, body: JSON.stringify(payload), redirect: "manual", dispatcher: proxyDispatcher(plan.proxyUrl),
}, undefined, signal);
} catch (error) {
const kind = classifyUpstreamFailure(error, { viaProxy: plan.proxyUrl !== undefined, externalAborted: signal.aborted });
loggerLog("warn", `[external-summary] candidate ${plan.model} attempt ${attempt}/${SUMMARY_MAX_ATTEMPTS} failed before response (${kind}): ${maskUrlsInText(error instanceof Error ? error.message : String(error))}`);
if (attempt < SUMMARY_MAX_ATTEMPTS && !signal.aborted && isFailFastUpstreamKind(kind)) {
await waitBeforeRetry(backoffDelayMs(attempt), signal);
continue;
}
throw error;
}
try {
if (!dispatched.response.ok) {
const snippet = await readErrorSnippet(dispatched.response);
// Mask only for the log line: the verdict below needs the raw body markers.
loggerLog("warn", `[external-summary] candidate ${plan.model} attempt ${attempt}/${SUMMARY_MAX_ATTEMPTS} rejected: HTTP ${dispatched.response.status} from ${host}${snippet ? ` — ${maskUrlsInText(snippet)}` : ""}`);
if (attempt < SUMMARY_MAX_ATTEMPTS && !signal.aborted && isTransientUpstreamError(dispatched.response.status, snippet)) {
await waitBeforeRetry(retryAfterDelayMs(dispatched.response.headers.get("retry-after"), backoffDelayMs(attempt)), signal);
continue;
}
throw new Error(`External summary upstream rejected the request with HTTP ${dispatched.response.status}`);
}
const result = parseSummary(plan.protocol, await readResponse(dispatched.response, maxResponseBytes));
signal.throwIfAborted();
return result;
} finally {
dispatched.clearTimer();
}
}
},
};
Expand Down
7 changes: 7 additions & 0 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ import { rulesEnabled, storeEffectiveRules } from "./rules-feature.js";
import { storeEffectiveSearchPlanAware } from "./decompress-shared.js";
import { armAutoFoldBackoff, autoFoldEngaged, AUTO_FOLD_BACKOFF_MS, externalQueueDroppedAll, withExternalSummaryTools } from "./external-summary-surface.js";
import { AUTO_FOLD_TARGET_MIN } from "./external-summary-settings.js";
import { auditStartupSummaryCredentials } from "./external-summary-credentials.js";
import { applyRanges } from "./stream.js";
import { attachSubagentSessions } from "./subagent-sessions.js";
import { buildSessionCacheReport, handleAcpCache, learnedImageReserve, noteClientAbort, readKeySwitchStats, readModelSwitchStats, settleUsageReport } from "./cache-ledger.js";
Expand Down Expand Up @@ -1136,6 +1137,12 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
);
}
}
// #3488: a broken external-summary credential used to surface only as a
// once-per-process warn at first dial; audit every effective enabled
// chain now so "which env var / secret is broken" is answerable at boot.
for (const problem of auditStartupSummaryCredentials()) {
log("warn", `[config] external summary credential: ${problem}`);
}
// #1723: residual zone drift is now an exception, not the norm — a
// lane'd launch landing ABOVE its preferred port means that port was
// held by something we must not wait on (foreign squatter, same-code
Expand Down
12 changes: 10 additions & 2 deletions src/web/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
} from "../config.js";
import { log } from "../logger.js";
import { validateHttpProxy } from "../upstream-proxy.js";
import { SummaryCredentialStore } from "../external-summary-credentials.js";
import { SummaryCredentialStore, type SummaryCredentialState } from "../external-summary-credentials.js";
import { parseExternalSummaryChain, expandExternalSummaryChain } from "../external-summary-settings.js";
import { agentProviderRecipes, agentRegistryStatus } from "../agent-providers.js";

Expand Down Expand Up @@ -91,6 +91,9 @@ export async function handleConfigGet(res: ServerResponse): Promise<void> {
const config = readConfig();
const rawCompress = config.compress && typeof config.compress === "object" ? config.compress as Record<string, unknown> : undefined;
const credentialStatus: Record<string, boolean> = {};
// #3488: additive five-state view (missing vs invalid value vs unreadable
// store) alongside the legacy boolean record scripts already consume.
const credentialDetail: Record<string, SummaryCredentialState> = {};
let hideInvalidSummary = false;
// Disabled chains skip strict target validation (an inert chain must not
// brick the config view), so surface credential status from the raw
Expand All @@ -112,7 +115,11 @@ export async function handleConfigGet(res: ServerResponse): Promise<void> {
? Object.values(loadNamedProviders()).flatMap((recipe) => recipe.credentialRef !== undefined ? [`secret:${recipe.credentialRef}`] : recipe.apiKeyEnv !== undefined ? [`env:${recipe.apiKeyEnv}`] : [])
: rawCredentialRefs(rawCompress.externalSummary);
const store = new SummaryCredentialStore();
for (const ref of refs) credentialStatus[ref] = store.configured(ref);
for (const ref of refs) {
const state = store.status(ref);
credentialStatus[ref] = state === "configured";
credentialDetail[ref] = state;
}
} catch {
// A manually edited invalid block can contain inline credentials.
// Never echo that block through either structured or raw config GET.
Expand All @@ -130,6 +137,7 @@ export async function handleConfigGet(res: ServerResponse): Promise<void> {
upstreamProxyMode: upstream.mode,
compress: config.compress ?? null,
externalSummaryCredentials: credentialStatus,
externalSummaryCredentialStatus: credentialDetail,
agentProviders: agentRegistryStatus(),
passthrough: passthroughState(process.env),
allowDshCompaction: allowDshCompactionState(process.env),
Expand Down
Loading
Loading