Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 34 additions & 12 deletions src/providers/circuit-breaker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ export class CircuitBreaker {
private failures = 0;
private lastFailureAt: number | null = null;
private openedAt: number | null = null;
private probeStartedAt: number | null = null;
private probeId = 0;
Comment thread
rohitg00 marked this conversation as resolved.
Resolved

private readonly failureThreshold: number;
private readonly failureWindowMs: number;
Expand All @@ -30,33 +32,53 @@ export class CircuitBreaker {
}

get isAllowed(): boolean {
if (this.state === "closed") return true;
return this.admit() !== null;
}

admit(): number | null {
if (this.state === "closed") return 0;
const now = Date.now();
if (this.state === "open") {
if (
this.openedAt &&
Date.now() - this.openedAt >= this.recoveryTimeoutMs
) {
this.state = "half-open";
return true;
if (!this.openedAt || now - this.openedAt < this.recoveryTimeoutMs) {
return null;
}
return false;
this.state = "half-open";
} else if (
this.probeStartedAt !== null &&
now - this.probeStartedAt < this.recoveryTimeoutMs
) {
return null;
}
return true;
this.probeStartedAt = now;
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
this.probeId += 1;
return this.probeId;
}

recordSuccess(): void {
if (this.state === "open") return;
private isStale(probe: number | undefined): boolean {
if (probe === undefined) return false;
return this.state === "half-open" ? probe !== this.probeId : probe !== 0;
}

recordSuccess(probe?: number): void {
if (this.state === "open" || this.isStale(probe)) return;
this.state = "closed";
this.failures = 0;
this.lastFailureAt = null;
this.openedAt = null;
this.probeStartedAt = null;
}

releaseProbe(probe: number): void {
if (this.state === "half-open" && probe === this.probeId) this.probeStartedAt = null;
}

recordFailure(): void {
recordFailure(probe?: number): void {
if (this.isStale(probe)) return;
const now = Date.now();
if (this.state === "half-open") {
this.state = "open";
this.openedAt = now;
this.probeStartedAt = null;
return;
}
if (this.lastFailureAt && now - this.lastFailureAt > this.failureWindowMs) {
Expand Down
11 changes: 7 additions & 4 deletions src/providers/resilient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@ export class ResilientProvider implements MemoryProvider {
}

private async call(fn: () => Promise<string>): Promise<string> {
if (!this.breaker.isAllowed) {
const probe = this.breaker.admit();
if (probe === null) {
throw new Error("circuit_breaker_open");
}
try {
const result = await fn();
this.breaker.recordSuccess();
this.breaker.recordSuccess(probe);
return result;
} catch (err) {
if (!isPayloadRejection(err)) {
this.breaker.recordFailure();
if (isPayloadRejection(err)) {
this.breaker.releaseProbe(probe);
} else {
this.breaker.recordFailure(probe);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
throw err;
}
Expand Down
109 changes: 109 additions & 0 deletions test/circuit-breaker.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,52 @@ describe("CircuitBreaker", () => {
expect(cb.getState().state).toBe("half-open");
});

it("lets only one probe through while half-open", () => {
const cb = new CircuitBreaker();
cb.recordFailure();
cb.recordFailure();
cb.recordFailure();
vi.advanceTimersByTime(30_000);
expect(cb.isAllowed).toBe(true);
expect(cb.isAllowed).toBe(false);
expect(cb.isAllowed).toBe(false);
cb.recordSuccess();
expect(cb.getState().state).toBe("closed");
expect(cb.isAllowed).toBe(true);
expect(cb.isAllowed).toBe(true);
});

it("reopens with a fresh openedAt when the probe fails", () => {
const cb = new CircuitBreaker();
cb.recordFailure();
cb.recordFailure();
cb.recordFailure();
vi.advanceTimersByTime(30_000);
expect(cb.isAllowed).toBe(true);
const probeAt = Date.now();
cb.recordFailure();
expect(cb.getState().state).toBe("open");
expect(cb.getState().openedAt).toBe(probeAt);
expect(cb.isAllowed).toBe(false);
vi.advanceTimersByTime(30_000);
expect(cb.isAllowed).toBe(true);
expect(cb.isAllowed).toBe(false);
});

it("releases a probe that never reports a result after the recovery timeout", () => {
const cb = new CircuitBreaker();
cb.recordFailure();
cb.recordFailure();
cb.recordFailure();
vi.advanceTimersByTime(30_000);
expect(cb.isAllowed).toBe(true);
vi.advanceTimersByTime(29_999);
expect(cb.isAllowed).toBe(false);
vi.advanceTimersByTime(1);
expect(cb.isAllowed).toBe(true);
expect(cb.isAllowed).toBe(false);
});

it("still opens after consecutive failures with no success between them", () => {
const cb = new CircuitBreaker();
cb.recordSuccess();
Expand All @@ -188,4 +234,67 @@ describe("CircuitBreaker", () => {
expect(cb.getState().state).toBe("open");
expect(cb.isAllowed).toBe(false);
});

function openAndExpireFirstProbe() {
const cb = new CircuitBreaker();
cb.recordFailure();
cb.recordFailure();
cb.recordFailure();
vi.advanceTimersByTime(30_000);
const first = cb.admit();
vi.advanceTimersByTime(30_000);
const second = cb.admit();
return { cb, first: first!, second: second! };
}

it("ignores a late failure from an expired probe while its replacement runs", () => {
const { cb, first, second } = openAndExpireFirstProbe();
expect(second).toBeGreaterThan(first);
cb.recordFailure(first);
expect(cb.getState().state).toBe("half-open");
cb.recordSuccess(second);
expect(cb.getState().state).toBe("closed");
});

it("ignores a late success from an expired probe", () => {
const { cb, first, second } = openAndExpireFirstProbe();
cb.recordSuccess(first);
expect(cb.getState().state).toBe("half-open");
expect(cb.admit()).toBeNull();
cb.recordFailure(second);
expect(cb.getState().state).toBe("open");
});

it("ignores an expired probe outcome after the current probe closed the breaker", () => {
const { cb, first, second } = openAndExpireFirstProbe();
cb.recordSuccess(second);
cb.recordFailure(first);
cb.recordFailure(first);
cb.recordFailure(first);
expect(cb.getState()).toMatchObject({ state: "closed", failures: 0 });
});

it("releaseProbe frees only the current half-open probe without changing counters", () => {
vi.useFakeTimers();
try {
const cb = new CircuitBreaker({ failureThreshold: 1, recoveryTimeoutMs: 1000 });
cb.recordFailure(cb.admit()!);
vi.advanceTimersByTime(1000);
const probe = cb.admit()!;
expect(cb.admit()).toBeNull();
cb.releaseProbe(probe + 1);
expect(cb.admit()).toBeNull();
cb.releaseProbe(probe);
expect(cb.getState().state).toBe("half-open");
expect(cb.admit()).toBe(probe + 1);
} finally {
vi.useRealTimers();
}
});

it("counts closed-state call outcomes as before", () => {
const cb = new CircuitBreaker();
for (let i = 0; i < 3; i++) cb.recordFailure(cb.admit()!);
expect(cb.getState().state).toBe("open");
});
});
30 changes: 29 additions & 1 deletion test/resilient-content-filter.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { ResilientProvider, isPayloadRejection } from "../src/providers/resilient.js";
import type { MemoryProvider } from "../src/types.js";

Expand Down Expand Up @@ -103,4 +103,32 @@ describe("ResilientProvider — content filter vs circuit breaker", () => {
expect(provider.circuitState.state).toBe("open");
await expect(provider.compress("sys", "x")).rejects.toThrow("circuit_breaker_open");
});

it("releases a half-open probe rejected by the content filter so the next call can probe", async () => {
vi.useFakeTimers();
try {
let mode: "down" | "filter" | "ok" = "down";
const provider = new ResilientProvider({
name: "stub",
async compress() {
if (mode === "down") throw new Error("OpenAI API error (503): upstream down");
if (mode === "filter") throw CONTENT_FILTER_ERROR;
return "<observation/>";
},
async summarize() {
return "<summary/>";
},
} as MemoryProvider);
for (let i = 0; i < 3; i++) await expect(provider.compress("sys", "x")).rejects.toThrow(/503/);
expect(provider.circuitState.state).toBe("open");
vi.advanceTimersByTime(30_000);
mode = "filter";
await expect(provider.compress("sys", "bad")).rejects.toThrow(/content_filter/);
mode = "ok";
await expect(provider.compress("sys", "fine")).resolves.toBe("<observation/>");
expect(provider.circuitState.state).toBe("closed");
} finally {
vi.useRealTimers();
}
});
});
Loading