diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index 0bce44dab7..cba4893068 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -15,6 +15,8 @@ export class CircuitBreaker { private failures = 0; private lastFailureAt: number | null = null; private openedAt: number | null = null; + private probeStartedAt: number | null = null; + private probeId = 0; private readonly failureThreshold: number; private readonly failureWindowMs: number; @@ -30,33 +32,53 @@ export class CircuitBreaker { } get isAllowed(): boolean { - if (this.state === "closed") return true; + return this.admit() !== null; + } + + admit(): number | null { + if (this.state === "closed") return 0; + const now = Date.now(); if (this.state === "open") { - if ( - this.openedAt && - Date.now() - this.openedAt >= this.recoveryTimeoutMs - ) { - this.state = "half-open"; - return true; + if (!this.openedAt || now - this.openedAt < this.recoveryTimeoutMs) { + return null; } - return false; + this.state = "half-open"; + } else if ( + this.probeStartedAt !== null && + now - this.probeStartedAt < this.recoveryTimeoutMs + ) { + return null; } - return true; + this.probeStartedAt = now; + this.probeId += 1; + return this.probeId; } - recordSuccess(): void { - if (this.state === "open") return; + private isStale(probe: number | undefined): boolean { + if (probe === undefined) return false; + return this.state === "half-open" ? probe !== this.probeId : probe !== 0; + } + + recordSuccess(probe?: number): void { + if (this.state === "open" || this.isStale(probe)) return; this.state = "closed"; this.failures = 0; this.lastFailureAt = null; this.openedAt = null; + this.probeStartedAt = null; + } + + releaseProbe(probe: number): void { + if (this.state === "half-open" && probe === this.probeId) this.probeStartedAt = null; } - recordFailure(): void { + recordFailure(probe?: number): void { + if (this.isStale(probe)) return; const now = Date.now(); if (this.state === "half-open") { this.state = "open"; this.openedAt = now; + this.probeStartedAt = null; return; } if (this.lastFailureAt && now - this.lastFailureAt > this.failureWindowMs) { diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index f82e578d72..202a3113a0 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -18,16 +18,19 @@ export class ResilientProvider implements MemoryProvider { } private async call(fn: () => Promise): Promise { - if (!this.breaker.isAllowed) { + const probe = this.breaker.admit(); + if (probe === null) { throw new Error("circuit_breaker_open"); } try { const result = await fn(); - this.breaker.recordSuccess(); + this.breaker.recordSuccess(probe); return result; } catch (err) { - if (!isPayloadRejection(err)) { - this.breaker.recordFailure(); + if (isPayloadRejection(err)) { + this.breaker.releaseProbe(probe); + } else { + this.breaker.recordFailure(probe); } throw err; } diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index 52cd6d0e76..c0aaf0aeb4 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -178,6 +178,52 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("half-open"); }); + it("lets only one probe through while half-open", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + expect(cb.isAllowed).toBe(false); + cb.recordSuccess(); + expect(cb.getState().state).toBe("closed"); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(true); + }); + + it("reopens with a fresh openedAt when the probe fails", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + const probeAt = Date.now(); + cb.recordFailure(); + expect(cb.getState().state).toBe("open"); + expect(cb.getState().openedAt).toBe(probeAt); + expect(cb.isAllowed).toBe(false); + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + }); + + it("releases a probe that never reports a result after the recovery timeout", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + vi.advanceTimersByTime(29_999); + expect(cb.isAllowed).toBe(false); + vi.advanceTimersByTime(1); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + }); + it("still opens after consecutive failures with no success between them", () => { const cb = new CircuitBreaker(); cb.recordSuccess(); @@ -188,4 +234,67 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("open"); expect(cb.isAllowed).toBe(false); }); + + function openAndExpireFirstProbe() { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + const first = cb.admit(); + vi.advanceTimersByTime(30_000); + const second = cb.admit(); + return { cb, first: first!, second: second! }; + } + + it("ignores a late failure from an expired probe while its replacement runs", () => { + const { cb, first, second } = openAndExpireFirstProbe(); + expect(second).toBeGreaterThan(first); + cb.recordFailure(first); + expect(cb.getState().state).toBe("half-open"); + cb.recordSuccess(second); + expect(cb.getState().state).toBe("closed"); + }); + + it("ignores a late success from an expired probe", () => { + const { cb, first, second } = openAndExpireFirstProbe(); + cb.recordSuccess(first); + expect(cb.getState().state).toBe("half-open"); + expect(cb.admit()).toBeNull(); + cb.recordFailure(second); + expect(cb.getState().state).toBe("open"); + }); + + it("ignores an expired probe outcome after the current probe closed the breaker", () => { + const { cb, first, second } = openAndExpireFirstProbe(); + cb.recordSuccess(second); + cb.recordFailure(first); + cb.recordFailure(first); + cb.recordFailure(first); + expect(cb.getState()).toMatchObject({ state: "closed", failures: 0 }); + }); + + it("releaseProbe frees only the current half-open probe without changing counters", () => { + vi.useFakeTimers(); + try { + const cb = new CircuitBreaker({ failureThreshold: 1, recoveryTimeoutMs: 1000 }); + cb.recordFailure(cb.admit()!); + vi.advanceTimersByTime(1000); + const probe = cb.admit()!; + expect(cb.admit()).toBeNull(); + cb.releaseProbe(probe + 1); + expect(cb.admit()).toBeNull(); + cb.releaseProbe(probe); + expect(cb.getState().state).toBe("half-open"); + expect(cb.admit()).toBe(probe + 1); + } finally { + vi.useRealTimers(); + } + }); + + it("counts closed-state call outcomes as before", () => { + const cb = new CircuitBreaker(); + for (let i = 0; i < 3; i++) cb.recordFailure(cb.admit()!); + expect(cb.getState().state).toBe("open"); + }); }); diff --git a/test/resilient-content-filter.test.ts b/test/resilient-content-filter.test.ts index 0c6026a18c..42ccd3a3ba 100644 --- a/test/resilient-content-filter.test.ts +++ b/test/resilient-content-filter.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { ResilientProvider, isPayloadRejection } from "../src/providers/resilient.js"; import type { MemoryProvider } from "../src/types.js"; @@ -103,4 +103,32 @@ describe("ResilientProvider — content filter vs circuit breaker", () => { expect(provider.circuitState.state).toBe("open"); await expect(provider.compress("sys", "x")).rejects.toThrow("circuit_breaker_open"); }); + + it("releases a half-open probe rejected by the content filter so the next call can probe", async () => { + vi.useFakeTimers(); + try { + let mode: "down" | "filter" | "ok" = "down"; + const provider = new ResilientProvider({ + name: "stub", + async compress() { + if (mode === "down") throw new Error("OpenAI API error (503): upstream down"); + if (mode === "filter") throw CONTENT_FILTER_ERROR; + return ""; + }, + async summarize() { + return ""; + }, + } as MemoryProvider); + for (let i = 0; i < 3; i++) await expect(provider.compress("sys", "x")).rejects.toThrow(/503/); + expect(provider.circuitState.state).toBe("open"); + vi.advanceTimersByTime(30_000); + mode = "filter"; + await expect(provider.compress("sys", "bad")).rejects.toThrow(/content_filter/); + mode = "ok"; + await expect(provider.compress("sys", "fine")).resolves.toBe(""); + expect(provider.circuitState.state).toBe("closed"); + } finally { + vi.useRealTimers(); + } + }); });