Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 2 additions & 20 deletions src/agent/stream-recovery.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

import { ProviderError, STREAM_STALL_MARKER } from "../llm/http.js";
import { rateLimitWaitMsFor } from "../llm/key-rotation.js";
import { isEmptyCompletionError } from "../llm/router.js";

export type StreamFailureKind =
Expand Down Expand Up @@ -222,18 +223,6 @@ function pick(delays: readonly number[], attempt: number, max: number): number {
return Math.min(delays[attempt] ?? delays[delays.length - 1] ?? 0, max);
}

function retryAfterSecondsFrom(error: unknown): number | undefined {
if (error instanceof ProviderError && error.retryAfterSeconds !== undefined) {
return error.retryAfterSeconds;
}
const match = (error instanceof Error ? error.message : String(error ?? "")).match(
/retry after ([0-9.]+)\s*s/i,
);
if (!match) return undefined;
const seconds = Number.parseFloat(match[1]!);
return Number.isFinite(seconds) && seconds >= 0 ? seconds : undefined;
}

export function planStreamRecovery(input: {
error?: unknown;
kind?: StreamFailureKind;
Expand Down Expand Up @@ -284,14 +273,7 @@ export function planStreamRecovery(input: {
case "rate-limit": {
const n = attempt(state.rateLimit, limits.maxRateLimit);
if (n >= limits.maxRateLimit) return giveUp;
const scheduled = pick([10_000, 20_000, 30_000, 40_000, 60_000], n, cap);
const providerSeconds = retryAfterSecondsFrom(input.error);
const delayMs = Math.min(
cap,
providerSeconds === undefined
? scheduled
: Math.max(scheduled, providerSeconds * 1000),
);
const delayMs = rateLimitWaitMsFor(input.error, n, cap);
return {
action: "retry",
kind,
Expand Down
45 changes: 45 additions & 0 deletions src/llm/key-rotation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,51 @@ export const MAX_PROVIDER_KEYS = 10;

export const MULTI_KEY_ATTEMPTS = 2;

export const RATE_LIMIT_RETRY_WAIT_MS: readonly number[] = [
10_000, 20_000, 30_000, 40_000, 60_000,
];

export function rateLimitRetryWaitMs(
attempt: number,
maxMs = Number.POSITIVE_INFINITY,
): number {
const index = Math.min(
Math.max(0, attempt),
RATE_LIMIT_RETRY_WAIT_MS.length - 1,
);
return Math.min(RATE_LIMIT_RETRY_WAIT_MS[index]!, maxMs);
}

export function providerRetryAfterMs(error: unknown): number | undefined {
if (error && typeof error === "object" && "retryAfterSeconds" in error) {
const seconds = (error as { retryAfterSeconds?: unknown })
.retryAfterSeconds;
if (
typeof seconds === "number" &&
Number.isFinite(seconds) &&
seconds >= 0
) {
return Math.ceil(seconds * 1000);
}
}
const match = errorMessage(error).match(/retry after ([0-9.]+)\s*s/i);
if (!match) return undefined;
const seconds = Number.parseFloat(match[1]!);
return Number.isFinite(seconds) && seconds >= 0
? Math.ceil(seconds * 1000)
: undefined;
}

export function rateLimitWaitMsFor(
error: unknown,
attempt: number,
maxMs = Number.POSITIVE_INFINITY,
): number {
const scheduled = rateLimitRetryWaitMs(attempt, maxMs);
const hint = providerRetryAfterMs(error);
return hint === undefined ? scheduled : Math.min(scheduled, hint);
}

export function buildKeyAttemptPlan(n: number, startIndex: number): number[] {
if (n <= 0) return [];
if (n === 1) return [0];
Expand Down
4 changes: 4 additions & 0 deletions src/llm/routing/error-classification.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
ProviderError,
STREAM_STALL_MARKER,
} from "../http.js";
import { rateLimitWaitMsFor } from "../key-rotation.js";
import { quotaOrRateLimited } from "../quota-signals.js";
import { resolveBuiltInProfile } from "../provider-profiles.js";
import { EFFORT_SCALE, nearestAcceptedEffort } from "../reasoning-controls.js";
Expand Down Expand Up @@ -193,6 +194,9 @@ export function isRetriableError(error: unknown): boolean {
}

export function retryWaitMs(error: unknown, attempt: number): number {
if (isRateLimited(error)) {
return rateLimitWaitMsFor(error, attempt, MAX_RETRY_WAIT_MS);
}
if (error instanceof ProviderError && error.retryAfterSeconds !== undefined) {
return Math.ceil(error.retryAfterSeconds * 1000);
}
Expand Down
1 change: 0 additions & 1 deletion src/mcp/auth/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,6 @@ class OAuthProvider implements McpAuthProvider {
}
throw new McpTransportError("network", "MCP OAuth authorization was declined.");
}
await this.requireConsent(metadata, scope);
const pkce = createPkcePair();
const loopback = this.deps.runLoopback ?? runLoopbackAuthorization;
const bootstrapClientId = this.clientId ?? this.deps.clientName ?? DEFAULT_CLIENT_NAME;
Expand Down
5 changes: 4 additions & 1 deletion src/ui-core/bootstrap/composition-root.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,10 @@ export function createCompositionRoot(
"plain",
);
if (!shown) {
sessionRef?.notice("info", lines.join(" Β· "));
sessionRef?.notice(
"info",
lines.filter((line) => line.trim().length > 0).join(" Β· "),
);
}
},
onAuthorizationUrl: (info) => {
Expand Down
48 changes: 32 additions & 16 deletions src/ui-core/commands/mcp-commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,12 +123,6 @@ async function addServer(
draft = input;
}
const written = await writeProjectMcpServer(draft);
if (written.ok) {
services.session.notice(
"info",
`added MCP server ${written.serverName} to ${written.displayPath} Β· connecting…`,
);
}
if (!written.ok) {
services.session.notice(
"warn",
Expand All @@ -145,16 +139,24 @@ async function addServer(
draft = retry;
continue;
}
const state = await services.mcp.refresh({ force: true });
const status = state.snapshot.statuses.find(
(candidate) => candidate.name === written.serverName,
);
if (status?.status === "ready") selectServer(services, status);
services.session.notice(
status?.status === "ready" ? "info" : "warn",
`${written.replaced ? "updated" : "added"} MCP server ${written.serverName} in ${written.displayPath}${status?.status === "ready" ? ` Β· use ${formatMcpToken(written.serverName)} in your prompt Β· ${status.toolCount} tool${status.toolCount === 1 ? "" : "s"}` : ` Β· ${status?.status ?? "not discovered"}${status?.detail ? ` Β· ${status.detail}` : ""}`}`,
const pendingToast = services.toast.show(
`adding MCP server ${written.serverName}…`,
{ level: "info", sticky: true },
);
return written.serverName;
try {
const state = await services.mcp.refresh({ force: true });
const status = state.snapshot.statuses.find(
(candidate) => candidate.name === written.serverName,
);
if (status?.status === "ready") selectServer(services, status);
services.session.notice(
status?.status === "ready" ? "info" : "warn",
`${written.replaced ? "updated" : "added"} MCP server ${written.serverName} in ${written.displayPath}${status?.status === "ready" ? ` Β· use ${formatMcpToken(written.serverName)} in your prompt Β· ${status.toolCount} tool${status.toolCount === 1 ? "" : "s"}` : ` Β· ${status?.status ?? "not discovered"}${status?.detail ? ` Β· ${status.detail}` : ""}`}`,
);
return written.serverName;
} finally {
services.toast.dismiss(pendingToast);
}
}
return undefined;
}
Expand Down Expand Up @@ -212,7 +214,21 @@ async function addKnownServer(
if (existing.status === "ready") selectServer(services, existing);
return;
}
services.session.notice("info", `adding ${known.title} MCP server…`);
const pendingToast = services.toast.show(
`adding ${known.title} MCP server…`,
{ level: "info", sticky: true },
);
try {
await finishKnownServerAdd(services, known);
} finally {
services.toast.dismiss(pendingToast);
}
}

async function finishKnownServerAdd(
services: AppServices,
known: KnownMcpServer,
): Promise<void> {
const collected: Record<string, string> = {};
for (const secret of known.secrets) {
if (secret.optional) continue;
Expand Down
23 changes: 8 additions & 15 deletions test/stream-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -188,34 +188,27 @@ describe("planStreamRecovery β€” bounded escalation", () => {
);
});

it("waits out a longer provider reset window instead of the schedule", () => {
it("caps provider reset windows at the uniform schedule but honors shorter hints", () => {
const state = createStreamRecoveryState();
const error = new ProviderError("rate limited", 429, "", 45);
const long = new ProviderError("rate limited", 429, "", 45);
expect(
planStreamRecovery({ kind: "rate-limit", state, error }).delayMs,
).toBe(45_000);
});

it("keeps the schedule as a floor and 60s as the ceiling for resets", () => {
const state = createStreamRecoveryState();
planStreamRecovery({ kind: "rate-limit", state, error: long }).delayMs,
).toBe(10_000);
recordRecoveryAttempt(state, "rate-limit");
const short = new ProviderError("rate limited", 429, "", 2);
expect(
planStreamRecovery({ kind: "rate-limit", state, error: short }).delayMs,
).toBe(10_000);
const long = new ProviderError("rate limited", 429, "", 600);
expect(
planStreamRecovery({ kind: "rate-limit", state, error: long }).delayMs,
).toBe(60_000);
).toBe(2_000);
});

it("derives the reset window from the failure message when needed", () => {
it("keeps the uniform schedule when the failure message carries a reset window", () => {
const state = createStreamRecoveryState();
const error = new Error(
"hetzner: Provider request failed with HTTP 429 (retry after 35s)",
);
expect(
planStreamRecovery({ kind: "rate-limit", state, error }).delayMs,
).toBe(35_000);
).toBe(10_000);
});

it("compacts on context overflow before retrying, then gives up", () => {
Expand Down