Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1320,6 +1320,10 @@ export function loadOptions(env: NodeJS.ProcessEnv = process.env): ProxyOptions
// is set so that proxy still wins (globalProxy is non-empty, so the short-circuit
// is skipped regardless).
const explicitDirect = proxyMode === "direct" && !biliProxy;
// #2809: unset mode resolves to direct too, but means "no preference" —
// this flag (unlike explicitDirect) never gates egress; it only lets
// failure diagnostics name the proxy the auto chain would have picked.
const defaultDirect = rawProxyMode === undefined && !biliProxy;
const proxy = biliProxy ?? (proxyMode === "direct" ? "" : proxyMode === "manual" ? webProxy ?? configProxy : configProxy);
const proxySource: ProxyOptions["proxySource"] = biliProxy
? "bili-env"
Expand All @@ -1342,6 +1346,7 @@ export function loadOptions(env: NodeJS.ProcessEnv = process.env): ProxyOptions
biliPort: port,
globalSource: proxySource,
explicitDirect,
defaultDirect,
};
// #1012: the launcher forwards the user's pre-strip proxy vars under
// BILI_INHERITED_* (the child's own env tier is intentionally empty —
Expand All @@ -1360,6 +1365,9 @@ export function loadOptions(env: NodeJS.ProcessEnv = process.env): ProxyOptions
// aux tier for every default-config user. Only an EXPLICIT "direct"
// mode (upstreamProxyMode / BILI_UPSTREAM_PROXY_MODE) opts aux egress
// out of the inherited tier; unset means "no preference".
// defaultDirect inherits via the spread above: whenever it is true,
// this explicitDirect is false, so the model-path short-circuit (and
// the #2809 diagnostic peek inside it) can never fire for aux.
explicitDirect: rawProxyMode === "direct" && !biliProxy,
...(httpProxy ? {} : inheritedHttpProxy ? { httpProxy: inheritedHttpProxy } : {}),
...(httpsProxy ? {} : inheritedHttpsProxy ? { httpsProxy: inheritedHttpsProxy } : {}),
Expand Down
8 changes: 6 additions & 2 deletions src/loop/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import { applyConfiguredCompression } from "../external-summary-compress.js";
import { executeSearchContextTarget, resolveDecompress } from "../decompress-shared.js";
import { toolFail, type ProxyToolResult } from "../proxy-tool-result.js";
import { fetchWithRetry, UpstreamHttpError } from "../fetch-util.js";
import { classifyUpstreamFailure, type UpstreamFailureKind } from "../upstream-fail.js";
import { classifyUpstreamFailure, type SkippedProxyInfo, type UpstreamFailureKind } from "../upstream-fail.js";
import { formatUpstreamError, proxyDispatcher } from "../upstream-proxy.js";
import { warnCacheCollapse } from "../cache-warn.js";
import { dumpRejectedBody } from "../error-dump.js";
Expand Down Expand Up @@ -172,6 +172,10 @@ export interface LoopCtx {
session: Session;
log: (msg: string) => void;
proxyUrl?: string;
/** #2809: diagnostic-only — the proxy the auto chain would have picked
* while default-direct mode ran this session's upstream directly. Names
* itself in re-request transport-failure logs like the outer forward does. */
skippedProxy?: SkippedProxyInfo;
/** #1536: origin (scheme://host) of the LLM endpoint this loop forwards to
* — part of the cache-invalidation target identity (rotating relays busts
* the prefix even for the same model+wire). Distinct from proxyUrl, which
Expand Down Expand Up @@ -1396,7 +1400,7 @@ export async function* runCompressLoop(
: absorbed ? "context work committed; " : "";
const msg = `${note}the follow-up request to the model failed (${TRANSPORT_FAILURE_LABELS[kind]}) before any response — your context is intact, resend your last message to continue`;
ctx.log(`[acp-loop] round ${round}: re-request transport failure (${kind}); ending turn in-band instead of throwing (#1453)`);
loggerLog("error", `[acp-loop] re-request transport failure after ${round} round(s): ${formatUpstreamError(e, requestOptions.url, ctx.proxyUrl)}`);
loggerLog("error", `[acp-loop] re-request transport failure after ${round} round(s): ${formatUpstreamError(e, requestOptions.url, ctx.proxyUrl, ctx.skippedProxy)}`);
yield adapter.emitError(msg);
return;
}
Expand Down
7 changes: 6 additions & 1 deletion src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { contextFromRegistry, loadRegistry, peekRegistryContext, peekRegistryOut
import { codexAlignedWindow } from "./codex-models.js";
import { fetchWithTimeout, MAX_REQUEST_BYTES, upstreamTimeoutMs } from "./fetch-util.js";
import { getUpstreamConnectionStatus, resolveProxy, resolveProxyDecision, proxyDispatcher, type UpstreamProxyDecision } from "./upstream-proxy.js";
import type { SkippedProxyInfo } from "./upstream-fail.js";
import { getUpstreamAlerts } from "./upstream-alerts.js";
import { hostIdForLog, maskHostPortForLog, setMaskHostsEnabled, maskUrlForLog, maskUrlsInText } from "./log-mask.js";
// Protocol codecs + the historical-image strip primitive live in the kernel now
Expand Down Expand Up @@ -1829,6 +1830,10 @@ type ForwardTarget = {
upstreamUrl: string;
headers: Record<string, string>;
proxyUrl: string | undefined;
/** #2809: diagnostic-only — the proxy the auto chain would have picked
* while default-direct mode ran this request directly. Rendered into
* failure surfaces (log/502/alert/loop re-request), never used for egress. */
skippedProxy?: SkippedProxyInfo;
};

// Log each unique (host, proxy, source) upstream-proxy decision once so the
Expand Down Expand Up @@ -1941,7 +1946,7 @@ export function buildForwardTarget(
}
const decision = resolveProxyDecision(opts.routes, opts.proxy, route?.rewrittenUrl ?? upstreamUrl, opts.proxyFallback);
logUpstreamProxyDecision(opts, upstreamUrl, decision);
return { upstreamUrl, headers, proxyUrl: decision.proxy };
return { upstreamUrl, headers, proxyUrl: decision.proxy, skippedProxy: decision.skippedProxy };
}

// #2228: model-decided nudge timing — the side-channel decision call. It
Expand Down
8 changes: 5 additions & 3 deletions src/server/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,9 @@ export async function handleAdminRoute(req: http.IncomingMessage, res: http.Serv
if (req.method === "POST" && req.url === "/__bili/upstream/test") {
const target = opts.upstream;
const targetUrl = new URL(target).origin;
const proxyUrl = resolveProxyDecision(opts.routes, opts.proxy, target, opts.proxyFallback).proxy;
// Keep the whole decision: #2809's skippedProxy diagnostic rides on it.
const decision = resolveProxyDecision(opts.routes, opts.proxy, target, opts.proxyFallback);
const proxyUrl = decision.proxy;
try {
const result = await fetchWithTimeout(targetUrl, {
method: "HEAD",
Expand All @@ -273,9 +275,9 @@ export async function handleAdminRoute(req: http.IncomingMessage, res: http.Serv
} catch (error) {
// #1682: probe failures deliberately do NOT feed the alert table —
// they are user-initiated diagnostics, not live-traffic evidence.
recordUpstreamConnection(targetUrl, proxyUrl, error);
recordUpstreamConnection(targetUrl, proxyUrl, error, decision.skippedProxy);
res.writeHead(502, { "content-type": "application/json" });
res.end(JSON.stringify({ error: formatUpstreamError(error, targetUrl, proxyUrl) }));
res.end(JSON.stringify({ error: formatUpstreamError(error, targetUrl, proxyUrl, decision.skippedProxy) }));
}
return;
}
Expand Down
10 changes: 5 additions & 5 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ export async function forward(
// #1757 resolved drop list, shared with wireTransform below (re-sent
// compress-retry bodies must carry the same drops as the initial forward).
let compatDropPaths: string[] = [];
const { upstreamUrl, headers, proxyUrl } = buildForwardTarget(req, opts, route, affinity, prepared !== null ? instanceId : undefined);
const { upstreamUrl, headers, proxyUrl, skippedProxy } = buildForwardTarget(req, opts, route, affinity, prepared !== null ? instanceId : undefined);
// #1884 re-sign arm: the native lane tunneled this request with the
// signing credential (x-bili-resign markers — stripped in
// buildForwardTarget, they must never reach the upstream). Every egress
Expand Down Expand Up @@ -501,15 +501,15 @@ export async function forward(
// #1682: any resolved response proves the host is reachable again.
clearUpstreamAlertsForHost(upstreamUrl);
} catch (error) {
recordUpstreamConnection(upstreamUrl, proxyUrl, error);
recordUpstreamAlert(upstreamUrl, error, proxyUrl !== undefined);
recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy);
recordUpstreamAlert(upstreamUrl, error, proxyUrl !== undefined, skippedProxy);
// #604: a network-level failure (socket reset, timeout abort) also never
// reports usage — arm the emergency shrink like the 5xx branch below.
if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "network failure", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate);
// [#1343] no response means the attached full text never reached the model —
// drop-and-log it (a corrective note surfaces on the next qualifying request).
if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream network failure");
throw new Error(`upstream request failed: ${formatUpstreamError(error, upstreamUrl, proxyUrl)}`, { cause: error });
throw new Error(`upstream request failed: ${formatUpstreamError(error, upstreamUrl, proxyUrl, skippedProxy)}`, { cause: error });
}
// #552 learn-on-failure: a converting upstream that rejects a role (codex
// ≥0.153 sends "developer"; vLLM/SGLang-style backends answer 400
Expand Down Expand Up @@ -1394,7 +1394,7 @@ export async function forward(
const loopBillingUpstream = route?.rewrittenUrl ?? (/^https?:\/\//i.test(req.url ?? "") ? req.url ?? undefined : opts.upstream);
const loop = runCompressLoop(
responseBody,
{ core, config, messages: prepared.processedMessages.length > 0 ? prepared.processedMessages : prepared.originalMessages, compressMessages: prepared.originalMessages, session: prepared.session, log: ctx.log, proxyUrl, upstreamOrigin: targetOrigin, protocol: prepared.protocol, textProtocol, debug: opts.debug, refreshFolded, visibilityMarkers, dumpSse: loopDumpDir ? (name, stream) => dumpStreamToFile(stream, loopDumpDir, name) : undefined, imageLearn: { host: upstreamHost(loopBillingUpstream), fp: `${imageBillingFor(opts, loopBillingUpstream)}:${imageTokenCapFor(opts, loopBillingUpstream)}` } },
{ core, config, messages: prepared.processedMessages.length > 0 ? prepared.processedMessages : prepared.originalMessages, compressMessages: prepared.originalMessages, session: prepared.session, log: ctx.log, proxyUrl, skippedProxy, upstreamOrigin: targetOrigin, protocol: prepared.protocol, textProtocol, debug: opts.debug, refreshFolded, visibilityMarkers, dumpSse: loopDumpDir ? (name, stream) => dumpStreamToFile(stream, loopDumpDir, name) : undefined, imageLearn: { host: upstreamHost(loopBillingUpstream), fp: `${imageBillingFor(opts, loopBillingUpstream)}:${imageTokenCapFor(opts, loopBillingUpstream)}` } },
parsedReq,
{ url: upstreamUrl, headers: reqHeaders, wireTransform, resign: applyResign },
adapter,
Expand Down
20 changes: 15 additions & 5 deletions src/upstream-alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* The web UI renders whatever is here as a global banner via the existing
* /__bili/overview poll; nothing else consumes this table. */

import { classifyUpstreamFailure, UPSTREAM_FAIL_HINTS, type UpstreamFailureKind } from "./upstream-fail.js";
import { classifyUpstreamFailure, hintWithSkippedProxy, redactedSkippedProxy, type SkippedProxyInfo, type UpstreamFailureKind } from "./upstream-fail.js";

export interface UpstreamAlert {
kind: UpstreamFailureKind;
Expand All @@ -22,6 +22,12 @@ export interface UpstreamAlert {
firstSeen: number;
lastSeen: number;
count: number;
/** #2809: set when the failing path ran default-direct while a proxy was
* available. The banner renders a LOCALIZED note from these structured
* fields (the client's i18n catalog owns the wording; `hint` above is
* the full English string for JSON consumers). Redacted. */
skippedProxy?: string;
skippedSource?: string;
}

/** v1 scope (#1682): only "fundamentally cannot connect" transport failures.
Expand Down Expand Up @@ -55,18 +61,22 @@ function keyFor(kind: UpstreamFailureKind, host: string): string {
}

/** Record one failed model request. Dedup key (kind, host): repeats refresh
* lastSeen/count only. Non-alert kinds are ignored entirely. */
export function recordUpstreamAlert(url: string, error: unknown, viaProxy: boolean): void {
* lastSeen/count only. Non-alert kinds are ignored entirely. `skipped` (#2809)
* rides along so the banner can name the proxy default-direct mode skipped. */
export function recordUpstreamAlert(url: string, error: unknown, viaProxy: boolean, skipped?: SkippedProxyInfo): void {
const kind = classifyUpstreamFailure(error, { viaProxy });
if (!ALERT_KINDS.has(kind)) return;
const host = hostOf(url);
const now = Date.now();
const key = keyFor(kind, host);
const skippedFields = skipped ? { skippedProxy: redactedSkippedProxy(skipped), skippedSource: skipped.source } : {};
const hint = hintWithSkippedProxy(kind, skipped);
const existing = alerts.get(key);
if (existing) {
existing.lastSeen = now;
existing.count += 1;
existing.hint = UPSTREAM_FAIL_HINTS[kind];
existing.hint = hint;
Object.assign(existing, skippedFields);
return;
}
if (alerts.size >= MAX_ALERTS) {
Expand All @@ -80,7 +90,7 @@ export function recordUpstreamAlert(url: string, error: unknown, viaProxy: boole
}
if (oldestKey !== undefined) alerts.delete(oldestKey);
}
alerts.set(key, { kind, host, hint: UPSTREAM_FAIL_HINTS[kind], firstSeen: now, lastSeen: now, count: 1 });
alerts.set(key, { kind, host, hint, firstSeen: now, lastSeen: now, count: 1, ...skippedFields });
}

/** Any resolved response to this host proves the path works again — clear ALL
Expand Down
46 changes: 46 additions & 0 deletions src/upstream-fail.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,3 +108,49 @@ export const UPSTREAM_FAIL_HINTS: Record<UpstreamFailureKind, string> = {
tls: "TLS/certificate failure at CONNECT or upstream handshake — CA/proxy MITM config",
unknown: "unclassified transport failure — report with full error chain",
};

/** Mask proxy credentials for display surfaces (log lines, 502 detail, the
* #2809 skipped-proxy note, alert payloads). Lives HERE instead of
* upstream-proxy so every renderer below that layer can share one
* implementation without a new import edge. */
export function redactProxyUrl(value: string | undefined): string | undefined {
if (!value) return undefined;
try {
const url = new URL(value);
if (url.username || url.password) {
url.username = "***";
url.password = "***";
}
return url.href;
} catch {
return "(invalid proxy URL)";
}
}

/** #2809: an AVAILABLE proxy that default-direct mode skipped for this host —
* the value the auto-detect chain would have resolved. Raw on purpose: each
* display channel redacts per its own needs (see redactedSkippedProxy). */
export type SkippedProxyInfo = {
proxy: string;
/** Tier that offered it: an env var name ("HTTPS_PROXY") or "windows-system". */
source: string;
};

/** Display form of a skipped proxy: credentials masked, the normalization
* trailing slash stripped so the note reads like a human wrote it. */
export function redactedSkippedProxy(skipped: SkippedProxyInfo): string {
return (redactProxyUrl(skipped.proxy) ?? "").replace(/\/$/, "");
}

/** #2809 dead-end: a fail-fast failure under UNSET proxy mode reads
* "unreachable/blackholed" while the machine's own env/system proxy could
* reach the host fine — and no channel ever named that proxy. This appends
* the conditional suffix naming it plus the opt-in paths. Only fail-fast
* kinds qualify: a post-connect failure may have run on a working path,
* where "skipped proxy" would mislead. Diagnostic text only — egress
* decisions never read it. */
export function hintWithSkippedProxy(kind: UpstreamFailureKind, skipped: SkippedProxyInfo | undefined): string {
const base = UPSTREAM_FAIL_HINTS[kind];
if (!skipped || !isFailFastUpstreamKind(kind)) return base;
return `${base}; available proxy ${redactedSkippedProxy(skipped)} (${skipped.source}) was skipped because upstreamProxyMode is unset (direct is the default by design, #346/#357) — follow it with "upstreamProxyMode": "auto" or BILI_UPSTREAM_PROXY_MODE=auto, or give this provider a "proxy" lane`;
}
Loading
Loading