Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 65 additions & 1 deletion src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,63 @@ function armFailureShrink(prepared: Prepared, log: (level: string, msg: string)
}
}

// #3434: upstream accepted the request then sent no bytes — the log shows
// nothing between `forward …` and the outcome (client disconnect / idle-watchdog
// abort), so an upstream stall reads as proxy-side. Arm a one-shot first-byte
// watchdog around the initial fetch: headers not arrived by the threshold → ONE
// warn naming target+method+lane. Additive logging only — deliberately NO
// fast-fail (would truncate long prefills, cf. #1706/#1714) and NO new config
// surface (code constant + test seam).
const UPSTREAM_FIRST_BYTE_WARN_MS = 30_000;
let firstByteWarnMs = UPSTREAM_FIRST_BYTE_WARN_MS;
export function _setFirstByteWarnMsForTest(ms?: number): void {
firstByteWarnMs = ms ?? UPSTREAM_FIRST_BYTE_WARN_MS;
}
// Dedupe once per origin per process (windowSourceLogged pattern) so a
// long-prefill local model logs at most one line, not one per slow request.
// Claimed at FIRE time (not arm time) so a fast request never suppresses a
// later genuine stall on the same origin.
const firstByteWarnedOrigins = new Set<string>();
export function _resetFirstByteWarnedOriginsForTest(): void {
firstByteWarnedOrigins.clear();
}
/**
* #3434: arm a one-shot watchdog logging ONE warn if the fetch behind `signal`
* is still pending after `firstByteWarnMs`. Returns the timer handle (the caller
* clears it when the fetch settles) or undefined when disarmed (threshold
* disabled, origin already warned, or signal already aborted).
*/
export function armFirstByteWatchdog(o: {
url: string;
method: string;
lane: string;
sessionId: string;
log: (level: string, msg: string) => void;
signal: AbortSignal;
}): ReturnType<typeof setTimeout> | undefined {
const threshold = firstByteWarnMs;
if (!Number.isFinite(threshold) || threshold <= 0) return undefined;
let origin: string;
try {
origin = new URL(o.url).origin;
} catch {
origin = o.url;
}
if (firstByteWarnedOrigins.has(origin)) return undefined;
if (o.signal.aborted) return undefined;
const seconds = Math.round(threshold / 1000);
const fire = (): void => {
// First-to-fire claims the origin; concurrent same-origin timers stay silent.
if (firstByteWarnedOrigins.has(origin)) return;
firstByteWarnedOrigins.add(origin);
o.log("warn", `[${o.sessionId}] upstream awaiting first byte ≥${seconds}s: ${o.method} ${maskUrlForLog(o.url)} (lane=${o.lane})`);
};
const timer = setTimeout(fire, threshold);
timer.unref();
o.signal.addEventListener("abort", () => clearTimeout(timer), { once: true });
return timer;
}

export async function forward(
req: http.IncomingMessage,
res: http.ServerResponse,
Expand Down Expand Up @@ -598,9 +655,16 @@ export async function forward(
}
let upstreamResult: Awaited<ReturnType<typeof fetchWithTimeout>>;
try {
upstreamResult = await fetchWithTransportRetry(upstreamUrl, init, undefined, clientAbort.signal, (info) => {
const lane = prepared?.sidePassthrough ? "side" : prepared ? "main" : "passthrough";
const fetchPromise = fetchWithTransportRetry(upstreamUrl, init, undefined, clientAbort.signal, (info) => {
log("warn", `[${prepared?.session.id ?? "unknown"}] [acp-proxy] upstream ${info.detail}; retrying in ${info.delayMs}ms (attempt ${info.attempt}/${info.maxAttempts})`);
});
const firstByteTimer = armFirstByteWatchdog({ url: upstreamUrl, method: req.method ?? "UNKNOWN", lane, sessionId: prepared?.session.id ?? "passthrough", log, signal: clientAbort.signal });
try {
upstreamResult = await fetchPromise;
} finally {
clearTimeout(firstByteTimer);
}
recordUpstreamConnection(upstreamUrl, proxyUrl);
// #1682: any resolved response proves the host is reachable again.
clearUpstreamAlertsForHost(upstreamUrl);
Expand Down
122 changes: 122 additions & 0 deletions tests/issue3434-first-byte-watchdog.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import assert from "node:assert/strict";
import { beforeEach, test } from "node:test";
import {
_resetFirstByteWarnedOriginsForTest,
_setFirstByteWarnMsForTest,
armFirstByteWatchdog,
} from "../src/server/relay.ts";

process.env.NODE_ENV = "test";

const THRESHOLD_MS = 30;
const WAIT_MS = 150;
const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));

type LogCall = { level: string; msg: string };
function makeLog(): { log: (level: string, msg: string) => void; calls: LogCall[] } {
const calls: LogCall[] = [];
return { log: (level, msg): void => { calls.push({ level, msg }); }, calls };
}

beforeEach(() => {
_resetFirstByteWarnedOriginsForTest();
_setFirstByteWarnMsForTest(THRESHOLD_MS);
});

test("warns exactly once when the fetch is still pending past the threshold", async () => {
const { log, calls } = makeLog();
const ac = new AbortController();
const timer = armFirstByteWatchdog({ url: "https://api.example.com/v1/messages", method: "POST", lane: "side", sessionId: "sess-1", log, signal: ac.signal });
assert.notEqual(timer, undefined);
await sleep(WAIT_MS);
if (timer) clearTimeout(timer);
assert.equal(calls.length, 1);
assert.equal(calls[0].level, "warn");
assert.ok(calls[0].msg.startsWith("[sess-1] upstream awaiting first byte ≥"), calls[0].msg);
// The target goes through maskUrlForLog — a non-allowlisted host is reduced
// to <private-host>, so the warn line must never leak the real upstream host.
assert.ok(calls[0].msg.includes(": POST https://<private-host>/v1/messages"), calls[0].msg);
assert.ok(calls[0].msg.endsWith("(lane=side)"), calls[0].msg);
});

test("does not warn when the fetch settles before the threshold (caller clears)", async () => {
const { log, calls } = makeLog();
const ac = new AbortController();
const timer = armFirstByteWatchdog({ url: "https://fast.example.com/v1/messages", method: "POST", lane: "main", sessionId: "s", log, signal: ac.signal });
assert.notEqual(timer, undefined);
clearTimeout(timer);
await sleep(WAIT_MS);
assert.equal(calls.length, 0);
});

test("dedupes per origin: a later pending request to the same origin stays silent", async () => {
const { log, calls } = makeLog();
const acA = new AbortController();
const tA = armFirstByteWatchdog({ url: "https://dup.example.com/x", method: "POST", lane: "main", sessionId: "a", log, signal: acA.signal });
await sleep(WAIT_MS);
if (tA) clearTimeout(tA);
assert.equal(calls.length, 1);
// Same origin (scheme+host+port), different path — disarmed at arm time.
const acB = new AbortController();
const tB = armFirstByteWatchdog({ url: "https://dup.example.com/y", method: "POST", lane: "main", sessionId: "b", log, signal: acB.signal });
assert.equal(tB, undefined);
await sleep(WAIT_MS);
assert.equal(calls.length, 1);
});

test("different origins each warn once (dedupe is keyed by origin, not global)", async () => {
const { log, calls } = makeLog();
const acA = new AbortController();
const tA = armFirstByteWatchdog({ url: "https://a.example.com/", method: "GET", lane: "main", sessionId: "a", log, signal: acA.signal });
const acB = new AbortController();
const tB = armFirstByteWatchdog({ url: "https://b.example.com/", method: "GET", lane: "main", sessionId: "b", log, signal: acB.signal });
await sleep(WAIT_MS);
if (tA) clearTimeout(tA);
if (tB) clearTimeout(tB);
assert.equal(calls.length, 2);
});

test("does not warn when the client aborts before the threshold", async () => {
const { log, calls } = makeLog();
const ac = new AbortController();
const timer = armFirstByteWatchdog({ url: "https://ab.example.com/", method: "POST", lane: "main", sessionId: "a", log, signal: ac.signal });
assert.notEqual(timer, undefined);
ac.abort();
await sleep(WAIT_MS);
assert.equal(calls.length, 0);
});

test("is disarmed when the signal was already aborted before arming", () => {
const { log, calls } = makeLog();
const ac = new AbortController();
ac.abort();
const timer = armFirstByteWatchdog({ url: "https://ab2.example.com/", method: "POST", lane: "main", sessionId: "a", log, signal: ac.signal });
assert.equal(timer, undefined);
assert.equal(calls.length, 0);
});

test("is disarmed when the threshold is disabled (<=0 or non-finite)", () => {
const { log, calls } = makeLog();
const ac = new AbortController();
_setFirstByteWarnMsForTest(0);
assert.equal(armFirstByteWatchdog({ url: "https://d.example.com/", method: "POST", lane: "main", sessionId: "a", log, signal: ac.signal }), undefined);
_setFirstByteWarnMsForTest(Number.NaN);
assert.equal(armFirstByteWatchdog({ url: "https://d.example.com/", method: "POST", lane: "main", sessionId: "a", log, signal: ac.signal }), undefined);
assert.equal(calls.length, 0);
});

test("_resetFirstByteWarnedOriginsForTest lets a previously-warned origin warn again", async () => {
const { log, calls } = makeLog();
const ac1 = new AbortController();
const t1 = armFirstByteWatchdog({ url: "https://rs.example.com/", method: "POST", lane: "main", sessionId: "a", log, signal: ac1.signal });
await sleep(WAIT_MS);
if (t1) clearTimeout(t1);
assert.equal(calls.length, 1);
_resetFirstByteWarnedOriginsForTest();
const ac2 = new AbortController();
const t2 = armFirstByteWatchdog({ url: "https://rs.example.com/", method: "POST", lane: "main", sessionId: "b", log, signal: ac2.signal });
assert.notEqual(t2, undefined);
await sleep(WAIT_MS);
if (t2) clearTimeout(t2);
assert.equal(calls.length, 2);
});
Loading