diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 5e1a91f57a..a3d4b60f59 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -611,10 +611,10 @@ function appendSpool(url, eventId, body, reason, options = {}) { if (bytes > policy.maxRecordBytes) return note("too-large", bytes); return withLock(paths.lock, (locked) => { let expired = 0; - if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); - const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes; - if (locked && over > 0) evictSent(paths, over); - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) { + if (locked && fileSize(paths.file) + claimBytes(paths) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); + const over = heldBytes(paths) + bytes - policy.maxBytes; + if (locked && over > 0 && over <= sentBytes(paths)) evictSent(paths, over); + if (heldBytes(paths) + bytes > policy.maxBytes) { if (expired) writeStats(paths, (s) => void (s.expired += expired)); return note("full", bytes); } @@ -666,6 +666,40 @@ function listSent(paths) { function sentBytes(paths) { return listSent(paths).reduce((n, s) => n + s.bytes, 0); } +function listClaims(paths) { + const claims = []; + let entries; + try { + entries = readdirSync(paths.dir); + } catch { + return claims; + } + const prefix = `${paths.name}.draining-`; + for (const entry of entries) { + if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; + const file = join(paths.dir, entry); + try { + const stat = statSync(file); + claims.push({ + file, + bytes: stat.size, + mtimeMs: stat.mtimeMs + }); + } catch {} + } + return claims; +} +function claimBytes(paths) { + return listClaims(paths).reduce((n, c) => n + c.bytes, 0); +} +const drainingClaims = /* @__PURE__ */ new Map(); +function retainedClaimBytes(paths) { + const draining = drainingClaims.get(paths.file); + return listClaims(paths).reduce((n, c) => draining?.has(c.file) ? n : n + c.bytes, 0); +} +function heldBytes(paths) { + return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths); +} function evictSent(paths, need) { let freed = 0; for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) { @@ -713,7 +747,7 @@ function retainSent(url, eventId, body, mark, options = {}) { const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; + if (fileSize(paths.file) + sentBytes(paths) + retainedClaimBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 384); try { writeSync(fd, line); @@ -757,17 +791,7 @@ function reconcileSent(url, bootId, options = {}) { return result; } function claimFiles(paths) { - const claimed = []; - try { - const prefix = `${paths.name}.draining-`; - for (const entry of readdirSync(paths.dir)) { - if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; - const full = join(paths.dir, entry); - try { - if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full); - } catch {} - } - } catch {} + const claimed = eligibleClaims(paths).map((claim) => claim.file); if (fileSize(paths.file) > 0) { const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`); withLock(paths.lock, (locked) => { @@ -799,8 +823,12 @@ function requeueUnlocked(paths, leftover) { utimesSync(target, 0, 0); } catch {} } +function eligibleClaims(paths, now = Date.now()) { + return listClaims(paths).filter((claim) => now - claim.mtimeMs > ORPHAN_CLAIM_MS); +} function spoolHasRecords(url, dir) { - return fileSize(spoolPaths(url, dir).file) > 0; + const paths = spoolPaths(url, dir); + return fileSize(paths.file) > 0 || eligibleClaims(paths).length > 0; } function drainInProgress(url, dir) { const paths = spoolPaths(url, dir); @@ -819,6 +847,7 @@ async function drainSpool(url, send, options = {}) { rejected: 0, expired: 0, corrupt: 0, + unreadable: 0, remaining: 0 }; const paths = spoolPaths(url, options.dir); @@ -840,12 +869,17 @@ async function drainSpool(url, send, options = {}) { skipped: "empty" }; const records = []; + const read = []; for (const file of files) try { const parsed = parseLines(readFileSync(file, "utf-8")); records.push(...parsed.records); result.corrupt += parsed.corrupt; - } catch {} + read.push(file); + } catch { + result.unreadable++; + } result.claimed = records.length; + drainingClaims.set(paths.file, new Set(read)); const seen = /* @__PURE__ */ new Set(); const now = Date.now(); const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY; @@ -888,7 +922,7 @@ async function drainSpool(url, send, options = {}) { } result.remaining = leftover.length; withLock(paths.lock, (locked) => locked ? requeueLocked(paths, leftover) : requeueUnlocked(paths, leftover), 2e3); - for (const file of files) try { + for (const file of read) try { unlinkSync(file); } catch {} writeStats(paths, (s) => { @@ -901,11 +935,13 @@ async function drainSpool(url, send, options = {}) { s.lastDrainDuplicates = result.duplicates; s.lastDrainRejected = result.rejected; s.lastDrainRemaining = result.remaining; + s.lastDrainUnreadable = result.unreadable; if (result.error) s.lastDrainError = result.error; else delete s.lastDrainError; }); return result; } finally { + drainingClaims.delete(paths.file); releaseLock(paths.drainLock); } } diff --git a/src/capture/spool.ts b/src/capture/spool.ts index ab7e571421..f1c802cf2e 100644 --- a/src/capture/spool.ts +++ b/src/capture/spool.ts @@ -62,6 +62,7 @@ export interface SpoolStats { lastDrainDuplicates?: number; lastDrainRejected?: number; lastDrainRemaining?: number; + lastDrainUnreadable?: number; lastDrainError?: string; } @@ -82,10 +83,10 @@ export interface SpoolSummary { oldestAt: string | null; retained: number; retainedBytes: number; + unreadable: number; maxBytes: number; maxAgeHours: number; stats: SpoolStats; - unreadable?: boolean; } export type SendOutcome = "delivered" | "duplicate" | "rejected" | "retry"; @@ -97,6 +98,7 @@ export interface DrainResult { rejected: number; expired: number; corrupt: number; + unreadable: number; remaining: number; error?: string; skipped?: "locked" | "empty" | "disabled"; @@ -339,10 +341,10 @@ export function appendSpool( if (bytes > policy.maxRecordBytes) return note("too-large", bytes); return withLock(paths.lock, (locked) => { let expired = 0; - if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); - const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes; - if (locked && over > 0) evictSent(paths, over); - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) { + if (locked && fileSize(paths.file) + claimBytes(paths) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy); + const over = heldBytes(paths) + bytes - policy.maxBytes; + if (locked && over > 0 && over <= sentBytes(paths)) evictSent(paths, over); + if (heldBytes(paths) + bytes > policy.maxBytes) { if (expired) writeStats(paths, (s) => void (s.expired += expired)); return note("full", bytes); } @@ -396,6 +398,47 @@ function sentBytes(paths: SpoolPaths): number { return listSent(paths).reduce((n, s) => n + s.bytes, 0); } +interface ClaimFile { + file: string; + bytes: number; + mtimeMs: number; +} + +function listClaims(paths: SpoolPaths): ClaimFile[] { + const claims: ClaimFile[] = []; + let entries: string[]; + try { + entries = readdirSync(paths.dir); + } catch { + return claims; + } + const prefix = `${paths.name}.draining-`; + for (const entry of entries) { + if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; + const file = join(paths.dir, entry); + try { + const stat = statSync(file); + claims.push({ file, bytes: stat.size, mtimeMs: stat.mtimeMs }); + } catch {} + } + return claims; +} + +function claimBytes(paths: SpoolPaths): number { + return listClaims(paths).reduce((n, c) => n + c.bytes, 0); +} + +const drainingClaims = new Map>(); + +function retainedClaimBytes(paths: SpoolPaths): number { + const draining = drainingClaims.get(paths.file); + return listClaims(paths).reduce((n, c) => (draining?.has(c.file) ? n : n + c.bytes), 0); +} + +function heldBytes(paths: SpoolPaths): number { + return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths); +} + function evictSent(paths: SpoolPaths, need: number): void { let freed = 0; for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) { @@ -448,7 +491,7 @@ export function retainSent( const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS; const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`); return withLock(paths.lock, () => { - if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false; + if (fileSize(paths.file) + sentBytes(paths) + retainedClaimBytes(paths) + bytes > policy.maxBytes) return false; const fd = openSync(file, "a", 0o600); try { writeSync(fd, line); @@ -496,17 +539,7 @@ export function reconcileSent( } function claimFiles(paths: SpoolPaths): string[] { - const claimed: string[] = []; - try { - const prefix = `${paths.name}.draining-`; - for (const entry of readdirSync(paths.dir)) { - if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue; - const full = join(paths.dir, entry); - try { - if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full); - } catch {} - } - } catch {} + const claimed = eligibleClaims(paths).map((claim) => claim.file); if (fileSize(paths.file) > 0) { const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`); withLock(paths.lock, (locked) => { @@ -541,8 +574,13 @@ function requeueUnlocked(paths: SpoolPaths, leftover: SpoolRecord[]): void { } catch {} } +function eligibleClaims(paths: SpoolPaths, now = Date.now()): ClaimFile[] { + return listClaims(paths).filter((claim) => now - claim.mtimeMs > ORPHAN_CLAIM_MS); +} + export function spoolHasRecords(url: string, dir?: string): boolean { - return fileSize(spoolPaths(url, dir).file) > 0; + const paths = spoolPaths(url, dir); + return fileSize(paths.file) > 0 || eligibleClaims(paths).length > 0; } export function drainInProgress(url: string, dir?: string): boolean { @@ -560,7 +598,7 @@ export async function drainSpool( options: { policy?: SpoolPolicy; dir?: string; maxRecords?: number; deadlineMs?: number } = {}, ): Promise { const policy = options.policy ?? spoolPolicy(); - const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 }; + const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, unreadable: 0, remaining: 0 }; const paths = spoolPaths(url, options.dir); if (!existsSync(paths.dir)) return { ...result, skipped: "empty" }; try { @@ -571,14 +609,19 @@ export async function drainSpool( const files = claimFiles(paths); if (files.length === 0) return { ...result, skipped: "empty" }; const records: SpoolRecord[] = []; + const read: string[] = []; for (const file of files) { try { const parsed = parseLines(readFileSync(file, "utf-8")); records.push(...parsed.records); result.corrupt += parsed.corrupt; - } catch {} + read.push(file); + } catch { + result.unreadable++; + } } result.claimed = records.length; + drainingClaims.set(paths.file, new Set(read)); const seen = new Set(); const now = Date.now(); const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY; @@ -618,7 +661,7 @@ export async function drainSpool( } result.remaining = leftover.length; withLock(paths.lock, (locked) => (locked ? requeueLocked(paths, leftover) : requeueUnlocked(paths, leftover)), 2000); - for (const file of files) { + for (const file of read) { try { unlinkSync(file); } catch {} @@ -633,11 +676,13 @@ export async function drainSpool( s.lastDrainDuplicates = result.duplicates; s.lastDrainRejected = result.rejected; s.lastDrainRemaining = result.remaining; + s.lastDrainUnreadable = result.unreadable; if (result.error) s.lastDrainError = result.error; else delete s.lastDrainError; }); return result; } finally { + drainingClaims.delete(paths.file); releaseLock(paths.drainLock); } } @@ -648,29 +693,28 @@ export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?: let records = 0; let bytes = 0; let oldestAt: string | null = null; - const files: string[] = []; - let unreadable = false; + let unreadable = 0; const noteError = (err: unknown) => { - if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") unreadable = true; + if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") unreadable++; }; - files.push(paths.file); try { - for (const entry of readdirSync(paths.dir)) { - if (entry.startsWith(`${paths.name}.draining-`) && entry.endsWith(".jsonl")) files.push(join(paths.dir, entry)); - } + readdirSync(paths.dir); } catch (err) { noteError(err); } + const files = [paths.file, ...listClaims(paths).map((claim) => claim.file)]; for (const file of files) { + let text: string; try { - const text = readFileSync(file, "utf-8"); - bytes += Buffer.byteLength(text); - for (const rec of parseLines(text).records) { - records++; - if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt; - } + text = readFileSync(file, "utf-8"); } catch (err) { noteError(err); + continue; + } + bytes += Buffer.byteLength(text); + for (const rec of parseLines(text).records) { + records++; + if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt; } } let retained = 0; @@ -690,9 +734,9 @@ export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?: oldestAt, retained, retainedBytes, + unreadable, maxBytes: policy.maxBytes, maxAgeHours: Math.round(policy.maxAgeMs / 3600_000), stats: readSpoolStats(paths), - ...(unreadable ? { unreadable: true } : {}), }; } diff --git a/src/cli/capture.ts b/src/cli/capture.ts index 99ea81e9e8..ceb89d3b65 100644 --- a/src/cli/capture.ts +++ b/src/cli/capture.ts @@ -82,6 +82,7 @@ export async function runCaptureCommand(options: CaptureCommandOptions): Promise log(` limits: ${spool.maxBytes} bytes, ${spool.maxAgeHours} h`); log(` totals: ${spool.stats.spooled} spooled, ${spool.stats.delivered} delivered, ${spool.stats.duplicates} duplicates, ${spool.stats.dropped} dropped, ${spool.stats.expired} expired`); if (spool.stats.lastDrainAt) log(` last drain: ${spool.stats.lastDrainAt}`); + if (spool.unreadable) log(` unreadable spool files kept for retry: ${spool.unreadable}`); if (spool.stats.lastDropAt) log(` last drop: ${spool.stats.lastDropAt} (${spool.stats.lastDropReason ?? "unknown"})`); const server = output["server"] as { diff --git a/src/functions/capture.ts b/src/functions/capture.ts index 63561e48c2..750dacaa7b 100644 --- a/src/functions/capture.ts +++ b/src/functions/capture.ts @@ -478,10 +478,11 @@ export function registerCaptureFunctions( rejected: acc.rejected + r.rejected, expired: acc.expired + r.expired, corrupt: acc.corrupt + r.corrupt, + unreadable: acc.unreadable + r.unreadable, remaining: acc.remaining + r.remaining, ...(r.error || acc.error ? { error: r.error ?? acc.error } : {}), }), - { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 }, + { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, unreadable: 0, remaining: 0 }, ); lastBootDrain = { ...total, at: new Date().toISOString() }; return results; diff --git a/test/capture-spool.test.ts b/test/capture-spool.test.ts index 238bdf8554..ebb9c49e68 100644 --- a/test/capture-spool.test.ts +++ b/test/capture-spool.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach } from "vitest"; -import { mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync, existsSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync, utimesSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -8,6 +8,7 @@ import { parseSentMark, reconcileSent, retainSent, + spoolHasRecords, spoolPaths, spoolPolicy, spoolSummary, @@ -97,11 +98,11 @@ describe("capture spool", () => { }); it("flags a spool it cannot read instead of reporting it empty", () => { - expect(spoolSummary(URL, { dir: join(dir, "missing"), policy }).unreadable).toBeUndefined(); + expect(spoolSummary(URL, { dir: join(dir, "missing"), policy }).unreadable).toBe(0); mkdirSync(spoolPaths(URL, dir).file, { recursive: true }); const summary = spoolSummary(URL, { dir, policy }); expect(summary.records).toBe(0); - expect(summary.unreadable).toBe(true); + expect(summary.unreadable).toBe(1); }); it("does nothing when the spool is disabled", () => { @@ -199,6 +200,134 @@ describe("capture spool", () => { expect(seen).toEqual(["evc_jjjjjjjjjjjj"]); expect(existsSync(orphan)).toBe(false); }); + + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "keeps a claimed file it cannot read and reports it", + async () => { + appendSpool(URL, "evc_kkkkkkkkkkkk", body("locked"), "timeout", { dir, policy }); + const paths = spoolPaths(URL, dir); + const blocked = join(dir, `${paths.name}.draining-99998-1.jsonl`); + writeFileSync(blocked, readFileSync(paths.file, "utf-8")); + writeFileSync(paths.file, ""); + appendSpool(URL, "evc_llllllllllll", body("readable"), "timeout", { dir, policy }); + const { chmodSync } = await import("node:fs"); + const old = new Date(Date.now() - 10 * 60_000); + utimesSync(blocked, old, old); + chmodSync(blocked, 0o000); + try { + const seen: string[] = []; + const result = await drainSpool(URL, async (rec) => (seen.push(rec.eventId), "delivered"), { dir, policy }); + expect(seen).toEqual(["evc_llllllllllll"]); + expect(result).toMatchObject({ delivered: 1, unreadable: 1 }); + expect(existsSync(blocked)).toBe(true); + expect(spoolSummary(URL, { dir, policy }).stats.lastDrainUnreadable).toBe(1); + expect(spoolSummary(URL, { dir, policy }).unreadable).toBe(1); + + const fresh = new Date(); + utimesSync(blocked, fresh, fresh); + appendSpool(URL, "evc_mmmmmmmmmmmm", body("later"), "timeout", { dir, policy }); + await drainSpool(URL, async () => "delivered", { dir, policy }); + const summary = spoolSummary(URL, { dir, policy }); + expect(summary.stats.lastDrainUnreadable).toBe(0); + expect(summary.unreadable).toBe(1); + + chmodSync(blocked, 0o600); + utimesSync(blocked, old, old); + seen.length = 0; + await drainSpool(URL, async (rec) => (seen.push(rec.eventId), "delivered"), { dir, policy }); + expect(seen).toEqual(["evc_kkkkkkkkkkkk"]); + expect(existsSync(blocked)).toBe(false); + } finally { + if (existsSync(blocked)) chmodSync(blocked, 0o600); + } + }, + ); +}); + +describe("capture spool retained claims", () => { + let dir: string; + const policy: SpoolPolicy = { enabled: true, maxBytes: 64 * 1024, maxAgeMs: 3600_000, maxRecordBytes: 16 * 1024 }; + + beforeEach(() => { + dir = join(mkdtempSync(join(tmpdir(), "am-spool-claims-")), "capture-spool"); + }); + + function writeClaim(name: string, content: string, ageMs: number): string { + const paths = spoolPaths(URL, dir); + mkdirSync(dir, { recursive: true }); + const claim = join(dir, `${paths.name}.draining-${name}.jsonl`); + writeFileSync(claim, content); + const at = new Date(Date.now() - ageMs); + utimesSync(claim, at, at); + return claim; + } + + it("reports records waiting when only an eligible retained claim is left", () => { + expect(spoolHasRecords(URL, dir)).toBe(false); + const claim = writeClaim("99997-1", "", 0); + writeFileSync(claim, `${JSON.stringify({ v: 1, eventId: "evc_nnnnnnnnnnnn", spooledAt: new Date().toISOString(), reason: "timeout", attempts: 0, body: body("n") })}\n`); + expect(spoolHasRecords(URL, dir)).toBe(false); + const old = new Date(Date.now() - 10 * 60_000); + utimesSync(claim, old, old); + expect(spoolHasRecords(URL, dir)).toBe(true); + }); + + it("counts retained claim bytes against the spool size limit and keeps the claim", () => { + const claim = writeClaim("99996-1", "x".repeat(policy.maxBytes - 100), 10 * 60_000); + const result = appendSpool(URL, "evc_oooooooooooo", body("over"), "timeout", { dir, policy }); + expect(result).toEqual({ spooled: false, dropped: "full" }); + expect(existsSync(claim)).toBe(true); + expect(existsSync(spoolPaths(URL, dir).file)).toBe(false); + }); + + it("keeps a delivered record's backup even when the claim being drained nearly fills the spool", async () => { + const now = new Date().toISOString(); + const records = Array.from({ length: 6 }, (_, i) => + `${JSON.stringify({ v: 1, eventId: `evc_drn${String(i).padStart(9, "0")}`, spooledAt: now, reason: "timeout", attempts: 0, body: body(`d${i}`, { pad: "p".repeat(10 * 1024) }) })}\n`).join(""); + writeClaim("99995-1", records, 10 * 60_000); + const kept: boolean[] = []; + const result = await drainSpool(URL, async (rec) => { + kept.push(retainSent(URL, rec.eventId, { hookType: "post_tool_use", sessionId: "ses_spool", marker: rec.eventId }, { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })); + return "delivered"; + }, { dir, policy }); + expect(result.delivered).toBe(6); + expect(kept).toEqual([true, true, true, true, true, true]); + }); + + it("counts a claim no drain is reading against the limit for sent backups", () => { + writeClaim("99996-1", "x".repeat(60 * 1024), 10 * 60_000); + let kept = 0; + for (let i = 0; i < 20; i++) { + if (retainSent(URL, `evc_u${String(i).padStart(11, "0")}`, body(`u${i}`), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })) kept++; + } + const held = readdirSync(dir).reduce((n, f) => n + statSync(join(dir, f)).size, 0); + expect(kept).toBeLessThan(20); + expect(held).toBeLessThanOrEqual(policy.maxBytes + 1024); + }); + + it("keeps sent backups when evicting them cannot make room for the new record", () => { + writeClaim("99994-1", "x".repeat(59 * 1024), 10 * 60_000); + let sentBytes = 0; + for (let i = 0; sentBytes < 4 * 1024; i++) { + expect(retainSent(URL, `evc_q${String(i).padStart(11, "0")}`, body(`q${i}`), { bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }, { dir, policy })).toBe(true); + sentBytes = readdirSync(dir).filter((f) => f.includes(".sent-")).reduce((n, f) => n + statSync(join(dir, f)).size, 0); + } + const result = appendSpool(URL, "evc_qqqqqqqqqqqq", body("big", { pad: "y".repeat(5 * 1024) }), "timeout", { dir, policy }); + expect(result).toEqual({ spooled: false, dropped: "full" }); + const after = readdirSync(dir).filter((f) => f.includes(".sent-")).reduce((n, f) => n + statSync(join(dir, f)).size, 0); + expect(after).toBe(sentBytes); + }); + + it("prunes expired live records when retained claims cause the overflow", () => { + writeClaim("99993-1", "x".repeat(60 * 1024), 10 * 60_000); + const old = new Date(Date.now() - 2 * policy.maxAgeMs).toISOString(); + const expired = Array.from({ length: 3 }, (_, i) => + `${JSON.stringify({ v: 1, eventId: `evc_exp${String(i).padStart(9, "0")}`, spooledAt: old, reason: "timeout", attempts: 0, body: body(`e${i}`, { pad: "z".repeat(800) }) })}\n`).join(""); + writeFileSync(spoolPaths(URL, dir).file, expired); + const result = appendSpool(URL, "evc_rrrrrrrrrrrr", body("fresh", { pad: "w".repeat(1500) }), "timeout", { dir, policy }); + expect(result).toEqual({ spooled: true }); + expect(lines(spoolPaths(URL, dir).file).map((r) => r.eventId)).toEqual(["evc_rrrrrrrrrrrr"]); + }); }); describe("capture spool retention until durable", () => {