From a1efe98db8661fc0a2f3e56312a0f7da082e087d Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Tue, 10 Nov 2026 17:25:42 +0530 Subject: [PATCH 1/4] fix(capture): keep spool files the drain could not read drainSpool claimed spool files, read them, and then unlinked every claimed file. A file that failed to read (EACCES, EBUSY, EMFILE) was swallowed by an empty catch and deleted with the rest, losing the observations in it. Unlink only the files that were read. An unreadable file stays as a claim file, which the next drain picks up once it is old enough. DrainResult reports the count as `unreadable`, the spool stats keep `lastDrainUnreadable`, and the capture CLI prints it with the spool totals. Fixes #1731 --- plugin/scripts/_capture.mjs | 10 ++++++++-- src/capture/spool.ts | 13 ++++++++++--- src/cli/capture.ts | 1 + src/functions/capture.ts | 3 ++- test/capture-spool.test.ts | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 5e1a91f57a..775460b285 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -819,6 +819,7 @@ async function drainSpool(url, send, options = {}) { rejected: 0, expired: 0, corrupt: 0, + unreadable: 0, remaining: 0 }; const paths = spoolPaths(url, options.dir); @@ -840,11 +841,15 @@ async function drainSpool(url, send, options = {}) { skipped: "empty" }; const records = []; + const read = []; for (const file of files) try { const parsed = parseLines(readFileSync(file, "utf-8")); records.push(...parsed.records); result.corrupt += parsed.corrupt; - } catch {} + read.push(file); + } catch { + result.unreadable++; + } result.claimed = records.length; const seen = /* @__PURE__ */ new Set(); const now = Date.now(); @@ -888,7 +893,7 @@ async function drainSpool(url, send, options = {}) { } result.remaining = leftover.length; withLock(paths.lock, (locked) => locked ? requeueLocked(paths, leftover) : requeueUnlocked(paths, leftover), 2e3); - for (const file of files) try { + for (const file of read) try { unlinkSync(file); } catch {} writeStats(paths, (s) => { @@ -901,6 +906,7 @@ async function drainSpool(url, send, options = {}) { s.lastDrainDuplicates = result.duplicates; s.lastDrainRejected = result.rejected; s.lastDrainRemaining = result.remaining; + s.lastDrainUnreadable = result.unreadable; if (result.error) s.lastDrainError = result.error; else delete s.lastDrainError; }); diff --git a/src/capture/spool.ts b/src/capture/spool.ts index e1e0f6c26a..f0961909e8 100644 --- a/src/capture/spool.ts +++ b/src/capture/spool.ts @@ -62,6 +62,7 @@ export interface SpoolStats { lastDrainDuplicates?: number; lastDrainRejected?: number; lastDrainRemaining?: number; + lastDrainUnreadable?: number; lastDrainError?: string; } @@ -96,6 +97,7 @@ export interface DrainResult { rejected: number; expired: number; corrupt: number; + unreadable: number; remaining: number; error?: string; skipped?: "locked" | "empty" | "disabled"; @@ -559,7 +561,7 @@ export async function drainSpool( options: { policy?: SpoolPolicy; dir?: string; maxRecords?: number; deadlineMs?: number } = {}, ): Promise { const policy = options.policy ?? spoolPolicy(); - const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 }; + const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, unreadable: 0, remaining: 0 }; const paths = spoolPaths(url, options.dir); if (!existsSync(paths.dir)) return { ...result, skipped: "empty" }; try { @@ -570,12 +572,16 @@ export async function drainSpool( const files = claimFiles(paths); if (files.length === 0) return { ...result, skipped: "empty" }; const records: SpoolRecord[] = []; + const read: string[] = []; for (const file of files) { try { const parsed = parseLines(readFileSync(file, "utf-8")); records.push(...parsed.records); result.corrupt += parsed.corrupt; - } catch {} + read.push(file); + } catch { + result.unreadable++; + } } result.claimed = records.length; const seen = new Set(); @@ -617,7 +623,7 @@ export async function drainSpool( } result.remaining = leftover.length; withLock(paths.lock, (locked) => (locked ? requeueLocked(paths, leftover) : requeueUnlocked(paths, leftover)), 2000); - for (const file of files) { + for (const file of read) { try { unlinkSync(file); } catch {} @@ -632,6 +638,7 @@ export async function drainSpool( s.lastDrainDuplicates = result.duplicates; s.lastDrainRejected = result.rejected; s.lastDrainRemaining = result.remaining; + s.lastDrainUnreadable = result.unreadable; if (result.error) s.lastDrainError = result.error; else delete s.lastDrainError; }); diff --git a/src/cli/capture.ts b/src/cli/capture.ts index 99ea81e9e8..1b132352ec 100644 --- a/src/cli/capture.ts +++ b/src/cli/capture.ts @@ -82,6 +82,7 @@ export async function runCaptureCommand(options: CaptureCommandOptions): Promise log(` limits: ${spool.maxBytes} bytes, ${spool.maxAgeHours} h`); log(` totals: ${spool.stats.spooled} spooled, ${spool.stats.delivered} delivered, ${spool.stats.duplicates} duplicates, ${spool.stats.dropped} dropped, ${spool.stats.expired} expired`); if (spool.stats.lastDrainAt) log(` last drain: ${spool.stats.lastDrainAt}`); + if (spool.stats.lastDrainUnreadable) log(` unreadable spool files kept for retry: ${spool.stats.lastDrainUnreadable}`); if (spool.stats.lastDropAt) log(` last drop: ${spool.stats.lastDropAt} (${spool.stats.lastDropReason ?? "unknown"})`); const server = output["server"] as { diff --git a/src/functions/capture.ts b/src/functions/capture.ts index 63561e48c2..750dacaa7b 100644 --- a/src/functions/capture.ts +++ b/src/functions/capture.ts @@ -478,10 +478,11 @@ export function registerCaptureFunctions( rejected: acc.rejected + r.rejected, expired: acc.expired + r.expired, corrupt: acc.corrupt + r.corrupt, + unreadable: acc.unreadable + r.unreadable, remaining: acc.remaining + r.remaining, ...(r.error || acc.error ? { error: r.error ?? acc.error } : {}), }), - { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 }, + { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, unreadable: 0, remaining: 0 }, ); lastBootDrain = { ...total, at: new Date().toISOString() }; return results; diff --git a/test/capture-spool.test.ts b/test/capture-spool.test.ts index 7deed3393a..03a1895c00 100644 --- a/test/capture-spool.test.ts +++ b/test/capture-spool.test.ts @@ -191,6 +191,39 @@ describe("capture spool", () => { expect(seen).toEqual(["evc_jjjjjjjjjjjj"]); expect(existsSync(orphan)).toBe(false); }); + + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "keeps a claimed file it cannot read and reports it", + async () => { + appendSpool(URL, "evc_kkkkkkkkkkkk", body("locked"), "timeout", { dir, policy }); + const paths = spoolPaths(URL, dir); + const blocked = join(dir, `${paths.name}.draining-99998-1.jsonl`); + writeFileSync(blocked, readFileSync(paths.file, "utf-8")); + writeFileSync(paths.file, ""); + appendSpool(URL, "evc_llllllllllll", body("readable"), "timeout", { dir, policy }); + const { chmodSync, utimesSync } = await import("node:fs"); + const old = new Date(Date.now() - 10 * 60_000); + utimesSync(blocked, old, old); + chmodSync(blocked, 0o000); + try { + const seen: string[] = []; + const result = await drainSpool(URL, async (rec) => (seen.push(rec.eventId), "delivered"), { dir, policy }); + expect(seen).toEqual(["evc_llllllllllll"]); + expect(result).toMatchObject({ delivered: 1, unreadable: 1 }); + expect(existsSync(blocked)).toBe(true); + expect(spoolSummary(URL, { dir, policy }).stats.lastDrainUnreadable).toBe(1); + + chmodSync(blocked, 0o600); + utimesSync(blocked, old, old); + seen.length = 0; + await drainSpool(URL, async (rec) => (seen.push(rec.eventId), "delivered"), { dir, policy }); + expect(seen).toEqual(["evc_kkkkkkkkkkkk"]); + expect(existsSync(blocked)).toBe(false); + } finally { + if (existsSync(blocked)) chmodSync(blocked, 0o600); + } + }, + ); }); describe("capture spool retention until durable", () => { From ba6f72cbba174324660dee96bed54081211de8d4 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 11 Nov 2026 12:12:59 +0530 Subject: [PATCH 2/4] fix(capture): drain, cap and report retained spool claims - A hook starts a drain when an eligible retained claim is waiting, not only when the live spool has records. - Append and sent-record capacity checks count retained claim bytes against the spool limit; unreadable claims are kept, not deleted. - Capture status reports retained unreadable claims on disk, separate from the last drain's count. --- plugin/scripts/_capture.mjs | 50 +++++++++++++++------- src/capture/spool.ts | 84 ++++++++++++++++++++++++------------- src/cli/capture.ts | 2 +- test/capture-spool.test.ts | 51 +++++++++++++++++++++- 4 files changed, 140 insertions(+), 47 deletions(-) diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 775460b285..9a5b87bf6f 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -612,9 +612,9 @@ function appendSpool(url, eventId, body, reason, options = {}) { return withLock(paths.lock, (locked) => { let expired = 0; if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); - const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes; + const over = heldBytes(paths) + bytes - policy.maxBytes; if (locked && over > 0) evictSent(paths, over); - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) { + if (heldBytes(paths) + bytes > policy.maxBytes) { if (expired) writeStats(paths, (s) => void (s.expired += expired)); return note("full", bytes); } @@ -666,6 +666,32 @@ function listSent(paths) { function sentBytes(paths) { return listSent(paths).reduce((n, s) => n + s.bytes, 0); } +function listClaims(paths) { + const claims = []; + let entries; + try { + entries = readdirSync(paths.dir); + } catch { + return claims; + } + const prefix = `${paths.name}.draining-`; + for (const entry of entries) { + if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; + const file = join(paths.dir, entry); + try { + const stat = statSync(file); + claims.push({ + file, + bytes: stat.size, + mtimeMs: stat.mtimeMs + }); + } catch {} + } + return claims; +} +function heldBytes(paths) { + return fileSize(paths.file) + sentBytes(paths) + listClaims(paths).reduce((n, c) => n + c.bytes, 0); +} function evictSent(paths, need) { let freed = 0; for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) { @@ -713,7 +739,7 @@ function retainSent(url, eventId, body, mark, options = {}) { const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; + if (heldBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 384); try { writeSync(fd, line); @@ -757,17 +783,7 @@ function reconcileSent(url, bootId, options = {}) { return result; } function claimFiles(paths) { - const claimed = []; - try { - const prefix = `${paths.name}.draining-`; - for (const entry of readdirSync(paths.dir)) { - if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; - const full = join(paths.dir, entry); - try { - if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full); - } catch {} - } - } catch {} + const claimed = eligibleClaims(paths).map((claim) => claim.file); if (fileSize(paths.file) > 0) { const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`); withLock(paths.lock, (locked) => { @@ -799,8 +815,12 @@ function requeueUnlocked(paths, leftover) { utimesSync(target, 0, 0); } catch {} } +function eligibleClaims(paths, now = Date.now()) { + return listClaims(paths).filter((claim) => now - claim.mtimeMs > ORPHAN_CLAIM_MS); +} function spoolHasRecords(url, dir) { - return fileSize(spoolPaths(url, dir).file) > 0; + const paths = spoolPaths(url, dir); + return fileSize(paths.file) > 0 || eligibleClaims(paths).length > 0; } function drainInProgress(url, dir) { const paths = spoolPaths(url, dir); diff --git a/src/capture/spool.ts b/src/capture/spool.ts index f0961909e8..aa72b3a687 100644 --- a/src/capture/spool.ts +++ b/src/capture/spool.ts @@ -83,6 +83,7 @@ export interface SpoolSummary { oldestAt: string | null; retained: number; retainedBytes: number; + unreadable: number; maxBytes: number; maxAgeHours: number; stats: SpoolStats; @@ -341,9 +342,9 @@ export function appendSpool( return withLock(paths.lock, (locked) => { let expired = 0; if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); - const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes; + const over = heldBytes(paths) + bytes - policy.maxBytes; if (locked && over > 0) evictSent(paths, over); - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) { + if (heldBytes(paths) + bytes > policy.maxBytes) { if (expired) writeStats(paths, (s) => void (s.expired += expired)); return note("full", bytes); } @@ -397,6 +398,36 @@ function sentBytes(paths: SpoolPaths): number { return listSent(paths).reduce((n, s) => n + s.bytes, 0); } +interface ClaimFile { + file: string; + bytes: number; + mtimeMs: number; +} + +function listClaims(paths: SpoolPaths): ClaimFile[] { + const claims: ClaimFile[] = []; + let entries: string[]; + try { + entries = readdirSync(paths.dir); + } catch { + return claims; + } + const prefix = `${paths.name}.draining-`; + for (const entry of entries) { + if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; + const file = join(paths.dir, entry); + try { + const stat = statSync(file); + claims.push({ file, bytes: stat.size, mtimeMs: stat.mtimeMs }); + } catch {} + } + return claims; +} + +function heldBytes(paths: SpoolPaths): number { + return fileSize(paths.file) + sentBytes(paths) + listClaims(paths).reduce((n, c) => n + c.bytes, 0); +} + function evictSent(paths: SpoolPaths, need: number): void { let freed = 0; for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) { @@ -449,7 +480,7 @@ export function retainSent( const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; + if (heldBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 0o600); try { writeSync(fd, line); @@ -497,17 +528,7 @@ export function reconcileSent( } function claimFiles(paths: SpoolPaths): string[] { - const claimed: string[] = []; - try { - const prefix = `${paths.name}.draining-`; - for (const entry of readdirSync(paths.dir)) { - if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; - const full = join(paths.dir, entry); - try { - if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full); - } catch {} - } - } catch {} + const claimed = eligibleClaims(paths).map((claim) => claim.file); if (fileSize(paths.file) > 0) { const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`); withLock(paths.lock, (locked) => { @@ -542,8 +563,13 @@ function requeueUnlocked(paths: SpoolPaths, leftover: SpoolRecord[]): void { } catch {} } +function eligibleClaims(paths: SpoolPaths, now = Date.now()): ClaimFile[] { + return listClaims(paths).filter((claim) => now - claim.mtimeMs > ORPHAN_CLAIM_MS); +} + export function spoolHasRecords(url: string, dir?: string): boolean { - return fileSize(spoolPaths(url, dir).file) > 0; + const paths = spoolPaths(url, dir); + return fileSize(paths.file) > 0 || eligibleClaims(paths).length > 0; } export function drainInProgress(url: string, dir?: string): boolean { @@ -654,22 +680,21 @@ export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?: let records = 0; let bytes = 0; let oldestAt: string | null = null; - const files: string[] = []; - if (existsSync(paths.file)) files.push(paths.file); - try { - for (const entry of readdirSync(paths.dir)) { - if (entry.startsWith(`${paths.name}.draining-`) && entry.endsWith(".jsonl")) files.push(join(paths.dir, entry)); - } - } catch {} + let unreadable = 0; + const files = [...(existsSync(paths.file) ? [paths.file] : []), ...listClaims(paths).map((claim) => claim.file)]; for (const file of files) { + let text: string; try { - const text = readFileSync(file, "utf-8"); - bytes += Buffer.byteLength(text); - for (const rec of parseLines(text).records) { - records++; - if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt; - } - } catch {} + text = readFileSync(file, "utf-8"); + } catch { + if (file !== paths.file) unreadable++; + continue; + } + bytes += Buffer.byteLength(text); + for (const rec of parseLines(text).records) { + records++; + if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt; + } } let retained = 0; let retainedBytes = 0; @@ -688,6 +713,7 @@ export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?: oldestAt, retained, retainedBytes, + unreadable, maxBytes: policy.maxBytes, maxAgeHours: Math.round(policy.maxAgeMs / 3600_000), stats: readSpoolStats(paths), diff --git a/src/cli/capture.ts b/src/cli/capture.ts index 1b132352ec..ceb89d3b65 100644 --- a/src/cli/capture.ts +++ b/src/cli/capture.ts @@ -82,7 +82,7 @@ export async function runCaptureCommand(options: CaptureCommandOptions): Promise log(` limits: ${spool.maxBytes} bytes, ${spool.maxAgeHours} h`); log(` totals: ${spool.stats.spooled} spooled, ${spool.stats.delivered} delivered, ${spool.stats.duplicates} duplicates, ${spool.stats.dropped} dropped, ${spool.stats.expired} expired`); if (spool.stats.lastDrainAt) log(` last drain: ${spool.stats.lastDrainAt}`); - if (spool.stats.lastDrainUnreadable) log(` unreadable spool files kept for retry: ${spool.stats.lastDrainUnreadable}`); + if (spool.unreadable) log(` unreadable spool files kept for retry: ${spool.unreadable}`); if (spool.stats.lastDropAt) log(` last drop: ${spool.stats.lastDropAt} (${spool.stats.lastDropReason ?? "unknown"})`); const server = output["server"] as { diff --git a/test/capture-spool.test.ts b/test/capture-spool.test.ts index 03a1895c00..b6e640025a 100644 --- a/test/capture-spool.test.ts +++ b/test/capture-spool.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach } from "vitest"; -import { mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync, existsSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync, utimesSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -8,6 +8,7 @@ import { parseSentMark, reconcileSent, retainSent, + spoolHasRecords, spoolPaths, spoolPolicy, spoolSummary, @@ -201,7 +202,7 @@ describe("capture spool", () => { writeFileSync(blocked, readFileSync(paths.file, "utf-8")); writeFileSync(paths.file, ""); appendSpool(URL, "evc_llllllllllll", body("readable"), "timeout", { dir, policy }); - const { chmodSync, utimesSync } = await import("node:fs"); + const { chmodSync } = await import("node:fs"); const old = new Date(Date.now() - 10 * 60_000); utimesSync(blocked, old, old); chmodSync(blocked, 0o000); @@ -212,6 +213,15 @@ describe("capture spool", () => { expect(result).toMatchObject({ delivered: 1, unreadable: 1 }); expect(existsSync(blocked)).toBe(true); expect(spoolSummary(URL, { dir, policy }).stats.lastDrainUnreadable).toBe(1); + expect(spoolSummary(URL, { dir, policy }).unreadable).toBe(1); + + const fresh = new Date(); + utimesSync(blocked, fresh, fresh); + appendSpool(URL, "evc_mmmmmmmmmmmm", body("later"), "timeout", { dir, policy }); + await drainSpool(URL, async () => "delivered", { dir, policy }); + const summary = spoolSummary(URL, { dir, policy }); + expect(summary.stats.lastDrainUnreadable).toBe(0); + expect(summary.unreadable).toBe(1); chmodSync(blocked, 0o600); utimesSync(blocked, old, old); @@ -226,6 +236,43 @@ describe("capture spool", () => { ); }); +describe("capture spool retained claims", () => { + let dir: string; + const policy: SpoolPolicy = { enabled: true, maxBytes: 64 * 1024, maxAgeMs: 3600_000, maxRecordBytes: 16 * 1024 }; + + beforeEach(() => { + dir = join(mkdtempSync(join(tmpdir(), "am-spool-claims-")), "capture-spool"); + }); + + function writeClaim(name: string, content: string, ageMs: number): string { + const paths = spoolPaths(URL, dir); + mkdirSync(dir, { recursive: true }); + const claim = join(dir, `${paths.name}.draining-${name}.jsonl`); + writeFileSync(claim, content); + const at = new Date(Date.now() - ageMs); + utimesSync(claim, at, at); + return claim; + } + + it("reports records waiting when only an eligible retained claim is left", () => { + expect(spoolHasRecords(URL, dir)).toBe(false); + const claim = writeClaim("99997-1", "", 0); + writeFileSync(claim, `${JSON.stringify({ v: 1, eventId: "evc_nnnnnnnnnnnn", spooledAt: new Date().toISOString(), reason: "timeout", attempts: 0, body: body("n") })}\n`); + expect(spoolHasRecords(URL, dir)).toBe(false); + const old = new Date(Date.now() - 10 * 60_000); + utimesSync(claim, old, old); + expect(spoolHasRecords(URL, dir)).toBe(true); + }); + + it("counts retained claim bytes against the spool size limit and keeps the claim", () => { + const claim = writeClaim("99996-1", "x".repeat(policy.maxBytes - 100), 10 * 60_000); + const result = appendSpool(URL, "evc_oooooooooooo", body("over"), "timeout", { dir, policy }); + expect(result).toEqual({ spooled: false, dropped: "full" }); + expect(existsSync(claim)).toBe(true); + expect(existsSync(spoolPaths(URL, dir).file)).toBe(false); + }); +}); + describe("capture spool retention until durable", () => { let dir: string; const policy: SpoolPolicy = { enabled: true, maxBytes: 64 * 1024, maxAgeMs: 3600_000, maxRecordBytes: 16 * 1024 }; From 22361cc1ed33797447a7a4b5965ec60f298987a1 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 11 Nov 2026 12:19:20 +0530 Subject: [PATCH 3/4] fix(capture): keep durability backups when retained claims fill the spool - retainSent checks only the live spool and sent backups, so a delivered record's backup is kept even while the claim being drained is large. - appendSpool evicts sent backups only when that alone can make room for the new record; otherwise it keeps them and rejects the record as full. - Expired live records are pruned when live, claimed and new bytes together exceed the limit, so a new capture is not dropped while pruning would free enough space. --- plugin/scripts/_capture.mjs | 11 +++++++---- src/capture/spool.ts | 12 ++++++++---- test/capture-spool.test.ts | 31 +++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 9a5b87bf6f..732bbdb2b2 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -611,9 +611,9 @@ function appendSpool(url, eventId, body, reason, options = {}) { if (bytes > policy.maxRecordBytes) return note("too-large", bytes); return withLock(paths.lock, (locked) => { let expired = 0; - if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); + if (locked && fileSize(paths.file) + claimBytes(paths) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); const over = heldBytes(paths) + bytes - policy.maxBytes; - if (locked && over > 0) evictSent(paths, over); + if (locked && over > 0 && over <= sentBytes(paths)) evictSent(paths, over); if (heldBytes(paths) + bytes > policy.maxBytes) { if (expired) writeStats(paths, (s) => void (s.expired += expired)); return note("full", bytes); @@ -689,8 +689,11 @@ function listClaims(paths) { } return claims; } +function claimBytes(paths) { + return listClaims(paths).reduce((n, c) => n + c.bytes, 0); +} function heldBytes(paths) { - return fileSize(paths.file) + sentBytes(paths) + listClaims(paths).reduce((n, c) => n + c.bytes, 0); + return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths); } function evictSent(paths, need) { let freed = 0; @@ -739,7 +742,7 @@ function retainSent(url, eventId, body, mark, options = {}) { const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (heldBytes(paths) + bytes > policy.maxBytes) return false; + if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 384); try { writeSync(fd, line); diff --git a/src/capture/spool.ts b/src/capture/spool.ts index aa72b3a687..74d232c012 100644 --- a/src/capture/spool.ts +++ b/src/capture/spool.ts @@ -341,9 +341,9 @@ export function appendSpool( if (bytes > policy.maxRecordBytes) return note("too-large", bytes); return withLock(paths.lock, (locked) => { let expired = 0; - if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); + if (locked && fileSize(paths.file) + claimBytes(paths) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); const over = heldBytes(paths) + bytes - policy.maxBytes; - if (locked && over > 0) evictSent(paths, over); + if (locked && over > 0 && over <= sentBytes(paths)) evictSent(paths, over); if (heldBytes(paths) + bytes > policy.maxBytes) { if (expired) writeStats(paths, (s) => void (s.expired += expired)); return note("full", bytes); @@ -424,8 +424,12 @@ function listClaims(paths: SpoolPaths): ClaimFile[] { return claims; } +function claimBytes(paths: SpoolPaths): number { + return listClaims(paths).reduce((n, c) => n + c.bytes, 0); +} + function heldBytes(paths: SpoolPaths): number { - return fileSize(paths.file) + sentBytes(paths) + listClaims(paths).reduce((n, c) => n + c.bytes, 0); + return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths); } function evictSent(paths: SpoolPaths, need: number): void { @@ -480,7 +484,7 @@ export function retainSent( const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (heldBytes(paths) + bytes > policy.maxBytes) return false; + if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 0o600); try { writeSync(fd, line); diff --git a/test/capture-spool.test.ts b/test/capture-spool.test.ts index b6e640025a..c664a77997 100644 --- a/test/capture-spool.test.ts +++ b/test/capture-spool.test.ts @@ -271,6 +271,37 @@ describe("capture spool retained claims", () => { expect(existsSync(claim)).toBe(true); expect(existsSync(spoolPaths(URL, dir).file)).toBe(false); }); + + it("keeps a delivered record's backup even when the claim being drained nearly fills the spool", () => { + writeClaim("99995-1", "x".repeat(policy.maxBytes - 200), 0); + const kept = retainSent(URL, "evc_ppppppppppp1", body("delivered"), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy }); + expect(kept).toBe(true); + expect(readdirSync(dir).some((f) => f.includes(".sent-"))).toBe(true); + }); + + it("keeps sent backups when evicting them cannot make room for the new record", () => { + writeClaim("99994-1", "x".repeat(60 * 1024), 10 * 60_000); + let sentBytes = 0; + for (let i = 0; sentBytes < 4 * 1024; i++) { + expect(retainSent(URL, `evc_q${String(i).padStart(11, "0")}`, body(`q${i}`), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })).toBe(true); + sentBytes = readdirSync(dir).filter((f) => f.includes(".sent-")).reduce((n, f) => n + statSync(join(dir, f)).size, 0); + } + const result = appendSpool(URL, "evc_qqqqqqqqqqqq", body("big", { pad: "y".repeat(5 * 1024) }), "timeout", { dir, policy }); + expect(result).toEqual({ spooled: false, dropped: "full" }); + const after = readdirSync(dir).filter((f) => f.includes(".sent-")).reduce((n, f) => n + statSync(join(dir, f)).size, 0); + expect(after).toBe(sentBytes); + }); + + it("prunes expired live records when retained claims cause the overflow", () => { + writeClaim("99993-1", "x".repeat(60 * 1024), 10 * 60_000); + const old = new Date(Date.now() - 2 * policy.maxAgeMs).toISOString(); + const expired = Array.from({ length: 3 }, (_, i) => + `${JSON.stringify({ v: 1, eventId: `evc_exp${String(i).padStart(9, "0")}`, spooledAt: old, reason: "timeout", attempts: 0, body: body(`e${i}`, { pad: "z".repeat(800) }) })}\n`).join(""); + writeFileSync(spoolPaths(URL, dir).file, expired); + const result = appendSpool(URL, "evc_rrrrrrrrrrrr", body("fresh", { pad: "w".repeat(1500) }), "timeout", { dir, policy }); + expect(result).toEqual({ spooled: true }); + expect(lines(spoolPaths(URL, dir).file).map((r) => r.eventId)).toEqual(["evc_rrrrrrrrrrrr"]); + }); }); describe("capture spool retention until durable", () => { From 6db4918037cc00521300382f71955ae9843e39b3 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 12 Nov 2026 17:23:14 +0530 Subject: [PATCH 4/4] fix(capture): count retained claims against the limit for sent backups retainSent left all claim files out of the size check, so an unreadable claim that stays on disk let sent backups grow the spool to nearly twice its limit. retainSent now counts every claim except the ones a drain in this process is delivering, so a delivered record still gets its backup while its own claim is on disk. Signed-off-by: Rohit Ghumare --- plugin/scripts/_capture.mjs | 9 ++++++++- src/capture/spool.ts | 11 ++++++++++- test/capture-spool.test.ts | 30 ++++++++++++++++++++++++------ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 732bbdb2b2..a3d4b60f59 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -692,6 +692,11 @@ function listClaims(paths) { function claimBytes(paths) { return listClaims(paths).reduce((n, c) => n + c.bytes, 0); } +const drainingClaims = /* @__PURE__ */ new Map(); +function retainedClaimBytes(paths) { + const draining = drainingClaims.get(paths.file); + return listClaims(paths).reduce((n, c) => draining?.has(c.file) ? n : n + c.bytes, 0); +} function heldBytes(paths) { return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths); } @@ -742,7 +747,7 @@ function retainSent(url, eventId, body, mark, options = {}) { const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; + if (fileSize(paths.file) + sentBytes(paths) + retainedClaimBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 384); try { writeSync(fd, line); @@ -874,6 +879,7 @@ async function drainSpool(url, send, options = {}) { result.unreadable++; } result.claimed = records.length; + drainingClaims.set(paths.file, new Set(read)); const seen = /* @__PURE__ */ new Set(); const now = Date.now(); const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY; @@ -935,6 +941,7 @@ async function drainSpool(url, send, options = {}) { }); return result; } finally { + drainingClaims.delete(paths.file); releaseLock(paths.drainLock); } } diff --git a/src/capture/spool.ts b/src/capture/spool.ts index 9875350dd1..f1c802cf2e 100644 --- a/src/capture/spool.ts +++ b/src/capture/spool.ts @@ -428,6 +428,13 @@ function claimBytes(paths: SpoolPaths): number { return listClaims(paths).reduce((n, c) => n + c.bytes, 0); } +const drainingClaims = new Map>(); + +function retainedClaimBytes(paths: SpoolPaths): number { + const draining = drainingClaims.get(paths.file); + return listClaims(paths).reduce((n, c) => (draining?.has(c.file) ? n : n + c.bytes), 0); +} + function heldBytes(paths: SpoolPaths): number { return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths); } @@ -484,7 +491,7 @@ export function retainSent( const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; + if (fileSize(paths.file) + sentBytes(paths) + retainedClaimBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 0o600); try { writeSync(fd, line); @@ -614,6 +621,7 @@ export async function drainSpool( } } result.claimed = records.length; + drainingClaims.set(paths.file, new Set(read)); const seen = new Set(); const now = Date.now(); const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY; @@ -674,6 +682,7 @@ export async function drainSpool( }); return result; } finally { + drainingClaims.delete(paths.file); releaseLock(paths.drainLock); } } diff --git a/test/capture-spool.test.ts b/test/capture-spool.test.ts index b822fa7083..ebb9c49e68 100644 --- a/test/capture-spool.test.ts +++ b/test/capture-spool.test.ts @@ -280,15 +280,33 @@ describe("capture spool retained claims", () => { expect(existsSync(spoolPaths(URL, dir).file)).toBe(false); }); - it("keeps a delivered record's backup even when the claim being drained nearly fills the spool", () => { - writeClaim("99995-1", "x".repeat(policy.maxBytes - 200), 0); - const kept = retainSent(URL, "evc_ppppppppppp1", body("delivered"), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy }); - expect(kept).toBe(true); - expect(readdirSync(dir).some((f) => f.includes(".sent-"))).toBe(true); + it("keeps a delivered record's backup even when the claim being drained nearly fills the spool", async () => { + const now = new Date().toISOString(); + const records = Array.from({ length: 6 }, (_, i) => + `${JSON.stringify({ v: 1, eventId: `evc_drn${String(i).padStart(9, "0")}`, spooledAt: now, reason: "timeout", attempts: 0, body: body(`d${i}`, { pad: "p".repeat(10 * 1024) }) })}\n`).join(""); + writeClaim("99995-1", records, 10 * 60_000); + const kept: boolean[] = []; + const result = await drainSpool(URL, async (rec) => { + kept.push(retainSent(URL, rec.eventId, { hookType: "post_tool_use", sessionId: "ses_spool", marker: rec.eventId }, { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })); + return "delivered"; + }, { dir, policy }); + expect(result.delivered).toBe(6); + expect(kept).toEqual([true, true, true, true, true, true]); + }); + + it("counts a claim no drain is reading against the limit for sent backups", () => { + writeClaim("99996-1", "x".repeat(60 * 1024), 10 * 60_000); + let kept = 0; + for (let i = 0; i < 20; i++) { + if (retainSent(URL, `evc_u${String(i).padStart(11, "0")}`, body(`u${i}`), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })) kept++; + } + const held = readdirSync(dir).reduce((n, f) => n + statSync(join(dir, f)).size, 0); + expect(kept).toBeLessThan(20); + expect(held).toBeLessThanOrEqual(policy.maxBytes + 1024); }); it("keeps sent backups when evicting them cannot make room for the new record", () => { - writeClaim("99994-1", "x".repeat(60 * 1024), 10 * 60_000); + writeClaim("99994-1", "x".repeat(59 * 1024), 10 * 60_000); let sentBytes = 0; for (let i = 0; sentBytes < 4 * 1024; i++) { expect(retainSent(URL, `evc_q${String(i).padStart(11, "0")}`, body(`q${i}`), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })).toBe(true);