From 20024f4716f90d7df45cedc42f0897e4d41d27bc Mon Sep 17 00:00:00 2001 From: reddeye1337 Date: Mon, 7 Sep 2026 21:42:24 +0300 Subject: [PATCH 1/3] fix(providers): clear the circuit breaker failure count on success recordSuccess only reset the counter when leaving half-open. In the closed state a success left it untouched, and recordFailure only cleared it when two failures were more than failureWindowMs apart. Under steady load a provider succeeding ~95 percent of the time still tripped, because each scattered failure landed inside the window opened by the previous one. Observed on a hosted proxy that returns the occasional 502: a handful of real upstream errors produced a stretch of 1,119 consecutive circuit_breaker_open fast-fails, and mem::compress reached 22,400 calls against 15,086 failures. Almost none of those failures were the provider being down. Counts failures since the last success instead, and moves the thresholds into config so a deployment behind a flaky upstream can widen them without a rebuild. Defaults go from 3 failures / 30s recovery to 10 / 15s: three scattered failures is not evidence that a hosted LLM proxy is unavailable. After the change, on the same install: 8 genuine upstream timeouts, 0 fast-fails. Adds a regression test that 100 calls at a 5 percent failure rate keep the breaker closed, and one that a genuinely failing provider still opens it. This is a different mechanism from #1259 and #1277, which filter which errors are counted as failures in resilient.ts; this fixes the counter never being reset by success in circuit-breaker.ts. The three compose. Signed-off-by: reddeye1337 --- src/config.ts | 32 ++++++++++++++++++++ src/providers/circuit-breaker.ts | 17 ++++++++--- src/providers/resilient.ts | 5 +++- test/circuit-breaker.test.ts | 51 ++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index a6c62ce55e..dd19a812ea 100644 --- a/src/config.ts +++ b/src/config.ts @@ -385,6 +385,38 @@ export function getGraphBatchSize(): number { return safeParseInt(getMergedEnv()["GRAPH_EXTRACTION_BATCH_SIZE"], 10); } +// Provider circuit breaker. The old hardcoded 3-failure trip was far +// too tight for a hosted LLM proxy that returns the occasional 502 or +// slow response: three scattered failures took the provider offline for +// every caller. Ten failures since the last success is still a decisive +// signal that the upstream is down, and a 15s recovery probe gets the +// provider back quickly when it is not. +const CIRCUIT_BREAKER_FAILURE_THRESHOLD = 10; +const CIRCUIT_BREAKER_FAILURE_WINDOW_MS = 60_000; +const CIRCUIT_BREAKER_RECOVERY_TIMEOUT_MS = 15_000; + +export function getCircuitBreakerOptions(): { + failureThreshold: number; + failureWindowMs: number; + recoveryTimeoutMs: number; +} { + const env = getMergedEnv(); + return { + failureThreshold: safeParseInt( + env["AGENTMEMORY_CIRCUIT_FAILURE_THRESHOLD"], + CIRCUIT_BREAKER_FAILURE_THRESHOLD, + ), + failureWindowMs: safeParseInt( + env["AGENTMEMORY_CIRCUIT_FAILURE_WINDOW_MS"], + CIRCUIT_BREAKER_FAILURE_WINDOW_MS, + ), + recoveryTimeoutMs: safeParseInt( + env["AGENTMEMORY_CIRCUIT_RECOVERY_TIMEOUT_MS"], + CIRCUIT_BREAKER_RECOVERY_TIMEOUT_MS, + ), + }; +} + // window for the smart-search followup-rate diagnostic. A second // search arriving within this many seconds (with disjoint results) // counts as a "follow-up" — a directional signal that the first result diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index e7f1f85d23..7e2c1ffb7d 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -1,6 +1,6 @@ import type { CircuitBreakerState } from "../types.js"; -interface CircuitBreakerOptions { +export interface CircuitBreakerOptions { failureThreshold?: number; failureWindowMs?: number; recoveryTimeoutMs?: number; @@ -47,10 +47,19 @@ export class CircuitBreaker { recordSuccess(): void { if (this.state === "half-open") { this.state = "closed"; - this.failures = 0; - this.lastFailureAt = null; - this.openedAt = null; } + // A success in the CLOSED state used to leave `failures` untouched, + // so the counter only ever reset when two consecutive failures were + // more than failureWindowMs apart. Under steady load — where + // failures are frequent enough to keep landing inside the window + // but rare in proportion to successes — the count crept to the + // threshold and opened the breaker on a provider that was mostly + // healthy. One flaky upstream then produced a thousand consecutive + // circuit_breaker_open fast-fails. Treat the window as "failures + // since the last success" and clear it here. + this.failures = 0; + this.lastFailureAt = null; + this.openedAt = null; } recordFailure(): void { diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 95ece40c96..ccf66d381b 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,8 +1,11 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; import { CircuitBreaker } from "./circuit-breaker.js"; +import { getCircuitBreakerOptions } from "../config.js"; export class ResilientProvider implements MemoryProvider { - private breaker = new CircuitBreaker(); + // Thresholds come from config so a deployment sitting behind a flaky + // proxy can widen them without a rebuild. + private breaker = new CircuitBreaker(getCircuitBreakerOptions()); name: string; constructor(private inner: MemoryProvider) { diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index a4e32a30a2..341286dc5c 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -104,4 +104,55 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("closed"); expect(cb.getState().failures).toBe(0); }); + + it("a success in closed state clears the accumulated failure count", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + expect(cb.getState().failures).toBe(2); + cb.recordSuccess(); + expect(cb.getState().failures).toBe(0); + expect(cb.getState().lastFailureAt).toBeNull(); + }); + + // Regression: the breaker counted failures since the last window + // expiry rather than since the last success, so a provider + // succeeding 95% of the time still tripped — each failure landed + // inside the 60s window opened by the previous one. In production + // that turned a handful of real upstream errors into ~1100 + // consecutive circuit_breaker_open fast-fails. + it("stays closed across 100 calls at a 5% failure rate", () => { + const cb = new CircuitBreaker(); + for (let i = 0; i < 100; i++) { + if (i % 20 === 0) cb.recordFailure(); + else cb.recordSuccess(); + vi.advanceTimersByTime(1_000); + expect(cb.getState().state).toBe("closed"); + } + expect(cb.isAllowed).toBe(true); + }); + + it("still opens on a genuinely failing provider", () => { + const cb = new CircuitBreaker({ failureThreshold: 10 }); + for (let i = 0; i < 10; i++) { + cb.recordFailure(); + vi.advanceTimersByTime(1_000); + } + expect(cb.getState().state).toBe("open"); + expect(cb.isAllowed).toBe(false); + }); + + it("honors configured thresholds and recovery timeout", () => { + const cb = new CircuitBreaker({ + failureThreshold: 10, + recoveryTimeoutMs: 15_000, + }); + for (let i = 0; i < 9; i++) cb.recordFailure(); + expect(cb.getState().state).toBe("closed"); + cb.recordFailure(); + expect(cb.getState().state).toBe("open"); + vi.advanceTimersByTime(15_000); + expect(cb.isAllowed).toBe(true); + expect(cb.getState().state).toBe("half-open"); + }); }); From ec23b4cf670e7643bec9f2140fb393872c4634d1 Mon Sep 17 00:00:00 2001 From: reddeye1337 Date: Mon, 7 Sep 2026 22:01:43 +0300 Subject: [PATCH 2/3] fix(providers): keep the recovery deadline when a success lands while open A request already in flight when another request opened the breaker still reaches recordSuccess. Clearing openedAt there left the breaker open with no recovery deadline, so isAllowed could never promote it to half-open and the provider stayed down permanently. Successes in the open state are now ignored. Also stops a non-positive env override from silently selecting the legacy 3 / 60s / 30s fallbacks: safeParseInt returns 0 and negatives verbatim, which CircuitBreaker rejects in favour of its own defaults rather than the configured ones. Both from review feedback on this PR. Signed-off-by: reddeye1337 --- src/config.ts | 14 +++++++++++--- src/providers/circuit-breaker.ts | 4 ++++ test/circuit-breaker.test.ts | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/config.ts b/src/config.ts index dd19a812ea..c9a1fd4224 100644 --- a/src/config.ts +++ b/src/config.ts @@ -395,6 +395,14 @@ const CIRCUIT_BREAKER_FAILURE_THRESHOLD = 10; const CIRCUIT_BREAKER_FAILURE_WINDOW_MS = 60_000; const CIRCUIT_BREAKER_RECOVERY_TIMEOUT_MS = 15_000; +function positiveEnvInt(value: string | undefined, fallback: number): number { + const parsed = safeParseInt(value, fallback); + // safeParseInt returns 0 or a negative number verbatim; CircuitBreaker + // rejects those and substitutes its own legacy fallbacks, so an override of + // "0" would silently select 3 / 60s / 30s rather than the values below. + return parsed > 0 ? parsed : fallback; +} + export function getCircuitBreakerOptions(): { failureThreshold: number; failureWindowMs: number; @@ -402,15 +410,15 @@ export function getCircuitBreakerOptions(): { } { const env = getMergedEnv(); return { - failureThreshold: safeParseInt( + failureThreshold: positiveEnvInt( env["AGENTMEMORY_CIRCUIT_FAILURE_THRESHOLD"], CIRCUIT_BREAKER_FAILURE_THRESHOLD, ), - failureWindowMs: safeParseInt( + failureWindowMs: positiveEnvInt( env["AGENTMEMORY_CIRCUIT_FAILURE_WINDOW_MS"], CIRCUIT_BREAKER_FAILURE_WINDOW_MS, ), - recoveryTimeoutMs: safeParseInt( + recoveryTimeoutMs: positiveEnvInt( env["AGENTMEMORY_CIRCUIT_RECOVERY_TIMEOUT_MS"], CIRCUIT_BREAKER_RECOVERY_TIMEOUT_MS, ), diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index 7e2c1ffb7d..058ed00b8a 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -45,6 +45,10 @@ export class CircuitBreaker { } recordSuccess(): void { + // A request already in flight when another request opened the breaker + // still lands here. Clearing openedAt in that state would strip the + // recovery deadline and leave the breaker open permanently. + if (this.state === "open") return; if (this.state === "half-open") { this.state = "closed"; } diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index 341286dc5c..46ed5e33fa 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -132,6 +132,27 @@ describe("CircuitBreaker", () => { expect(cb.isAllowed).toBe(true); }); + // An in-flight request can succeed after a different request has already + // opened the breaker. That success must not strip the recovery deadline. + it("keeps openedAt when a success arrives while open", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + expect(cb.getState().state).toBe("open"); + const openedAt = cb.getState().openedAt; + + cb.recordSuccess(); + + expect(cb.getState().state).toBe("open"); + expect(cb.getState().openedAt).toBe(openedAt); + expect(cb.isAllowed).toBe(false); + + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + expect(cb.getState().state).toBe("half-open"); + }); + it("still opens on a genuinely failing provider", () => { const cb = new CircuitBreaker({ failureThreshold: 10 }); for (let i = 0; i < 10; i++) { From 0030609ae91badf150acdbbac4eec771119094a4 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sun, 8 Nov 2026 21:48:50 +0530 Subject: [PATCH 3/3] fix(providers): reset circuit breaker failure count on success A success in the closed state left the failure count in place, so a mostly healthy provider with scattered failures inside the window eventually opened the breaker. Clear the count on every success outside the open state, and ignore successes that land while open so the recovery deadline is kept. Drop the default threshold change and the new circuit breaker env vars to keep this a minimal bugfix. --- src/config.ts | 40 -------------------------------- src/providers/circuit-breaker.ts | 18 ++------------ src/providers/resilient.ts | 5 +--- test/circuit-breaker.test.ts | 39 +++++++++++-------------------- 4 files changed, 17 insertions(+), 85 deletions(-) diff --git a/src/config.ts b/src/config.ts index 35b392848b..53823f19c7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -524,46 +524,6 @@ export function getGraphBatchSize(): number { return safeParseInt(getMergedEnv()["GRAPH_EXTRACTION_BATCH_SIZE"], 10); } -// Provider circuit breaker. The old hardcoded 3-failure trip was far -// too tight for a hosted LLM proxy that returns the occasional 502 or -// slow response: three scattered failures took the provider offline for -// every caller. Ten failures since the last success is still a decisive -// signal that the upstream is down, and a 15s recovery probe gets the -// provider back quickly when it is not. -const CIRCUIT_BREAKER_FAILURE_THRESHOLD = 10; -const CIRCUIT_BREAKER_FAILURE_WINDOW_MS = 60_000; -const CIRCUIT_BREAKER_RECOVERY_TIMEOUT_MS = 15_000; - -function positiveEnvInt(value: string | undefined, fallback: number): number { - const parsed = safeParseInt(value, fallback); - // safeParseInt returns 0 or a negative number verbatim; CircuitBreaker - // rejects those and substitutes its own legacy fallbacks, so an override of - // "0" would silently select 3 / 60s / 30s rather than the values below. - return parsed > 0 ? parsed : fallback; -} - -export function getCircuitBreakerOptions(): { - failureThreshold: number; - failureWindowMs: number; - recoveryTimeoutMs: number; -} { - const env = getMergedEnv(); - return { - failureThreshold: positiveEnvInt( - env["AGENTMEMORY_CIRCUIT_FAILURE_THRESHOLD"], - CIRCUIT_BREAKER_FAILURE_THRESHOLD, - ), - failureWindowMs: positiveEnvInt( - env["AGENTMEMORY_CIRCUIT_FAILURE_WINDOW_MS"], - CIRCUIT_BREAKER_FAILURE_WINDOW_MS, - ), - recoveryTimeoutMs: positiveEnvInt( - env["AGENTMEMORY_CIRCUIT_RECOVERY_TIMEOUT_MS"], - CIRCUIT_BREAKER_RECOVERY_TIMEOUT_MS, - ), - }; -} - // window for the smart-search followup-rate diagnostic. A second // search arriving within this many seconds (with disjoint results) // counts as a "follow-up" — a directional signal that the first result diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index 15a9067cfb..0bce44dab7 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -1,6 +1,6 @@ import type { CircuitBreakerState } from "../types.js"; -export interface CircuitBreakerOptions { +interface CircuitBreakerOptions { failureThreshold?: number; failureWindowMs?: number; recoveryTimeoutMs?: number; @@ -45,22 +45,8 @@ export class CircuitBreaker { } recordSuccess(): void { - // A request already in flight when another request opened the breaker - // still lands here. Clearing openedAt in that state would strip the - // recovery deadline and leave the breaker open permanently. if (this.state === "open") return; - if (this.state === "half-open") { - this.state = "closed"; - } - // A success in the CLOSED state used to leave `failures` untouched, - // so the counter only ever reset when two consecutive failures were - // more than failureWindowMs apart. Under steady load — where - // failures are frequent enough to keep landing inside the window - // but rare in proportion to successes — the count crept to the - // threshold and opened the breaker on a provider that was mostly - // healthy. One flaky upstream then produced a thousand consecutive - // circuit_breaker_open fast-fails. Treat the window as "failures - // since the last success" and clear it here. + this.state = "closed"; this.failures = 0; this.lastFailureAt = null; this.openedAt = null; diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index ccf66d381b..95ece40c96 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,11 +1,8 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; import { CircuitBreaker } from "./circuit-breaker.js"; -import { getCircuitBreakerOptions } from "../config.js"; export class ResilientProvider implements MemoryProvider { - // Thresholds come from config so a deployment sitting behind a flaky - // proxy can widen them without a rebuild. - private breaker = new CircuitBreaker(getCircuitBreakerOptions()); + private breaker = new CircuitBreaker(); name: string; constructor(private inner: MemoryProvider) { diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index 36b85b91b5..52cd6d0e76 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -138,12 +138,6 @@ describe("CircuitBreaker", () => { expect(cb.getState().lastFailureAt).toBeNull(); }); - // Regression: the breaker counted failures since the last window - // expiry rather than since the last success, so a provider - // succeeding 95% of the time still tripped — each failure landed - // inside the 60s window opened by the previous one. In production - // that turned a handful of real upstream errors into ~1100 - // consecutive circuit_breaker_open fast-fails. it("stays closed across 100 calls at a 5% failure rate", () => { const cb = new CircuitBreaker(); for (let i = 0; i < 100; i++) { @@ -155,8 +149,16 @@ describe("CircuitBreaker", () => { expect(cb.isAllowed).toBe(true); }); - // An in-flight request can succeed after a different request has already - // opened the breaker. That success must not strip the recovery deadline. + it("stays closed when failures are interleaved with successes inside the window", () => { + const cb = new CircuitBreaker(); + for (let i = 0; i < 5; i++) { + cb.recordFailure(); + cb.recordSuccess(); + } + expect(cb.getState().state).toBe("closed"); + expect(cb.getState().failures).toBe(0); + }); + it("keeps openedAt when a success arrives while open", () => { const cb = new CircuitBreaker(); cb.recordFailure(); @@ -176,27 +178,14 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("half-open"); }); - it("still opens on a genuinely failing provider", () => { - const cb = new CircuitBreaker({ failureThreshold: 10 }); - for (let i = 0; i < 10; i++) { + it("still opens after consecutive failures with no success between them", () => { + const cb = new CircuitBreaker(); + cb.recordSuccess(); + for (let i = 0; i < 3; i++) { cb.recordFailure(); vi.advanceTimersByTime(1_000); } expect(cb.getState().state).toBe("open"); expect(cb.isAllowed).toBe(false); }); - - it("honors configured thresholds and recovery timeout", () => { - const cb = new CircuitBreaker({ - failureThreshold: 10, - recoveryTimeoutMs: 15_000, - }); - for (let i = 0; i < 9; i++) cb.recordFailure(); - expect(cb.getState().state).toBe("closed"); - cb.recordFailure(); - expect(cb.getState().state).toBe("open"); - vi.advanceTimersByTime(15_000); - expect(cb.isAllowed).toBe(true); - expect(cb.getState().state).toBe("half-open"); - }); });