Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 55 additions & 19 deletions plugin/scripts/_capture.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -611,10 +611,10 @@ function appendSpool(url, eventId, body, reason, options = {}) {
if (bytes > policy.maxRecordBytes) return note("too-large", bytes);
return withLock(paths.lock, (locked) => {
let expired = 0;
if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy);
const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes;
if (locked && over > 0) evictSent(paths, over);
if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) {
if (locked && fileSize(paths.file) + claimBytes(paths) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy);
const over = heldBytes(paths) + bytes - policy.maxBytes;
if (locked && over > 0 && over <= sentBytes(paths)) evictSent(paths, over);
if (heldBytes(paths) + bytes > policy.maxBytes) {
if (expired) writeStats(paths, (s) => void (s.expired += expired));
return note("full", bytes);
}
Expand Down Expand Up @@ -666,6 +666,40 @@ function listSent(paths) {
function sentBytes(paths) {
return listSent(paths).reduce((n, s) => n + s.bytes, 0);
}
function listClaims(paths) {
const claims = [];
let entries;
try {
entries = readdirSync(paths.dir);
} catch {
return claims;
}
const prefix = `${paths.name}.draining-`;
for (const entry of entries) {
if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue;
const file = join(paths.dir, entry);
try {
const stat = statSync(file);
claims.push({
file,
bytes: stat.size,
mtimeMs: stat.mtimeMs
});
} catch {}
}
return claims;
}
function claimBytes(paths) {
return listClaims(paths).reduce((n, c) => n + c.bytes, 0);
}
const drainingClaims = /* @__PURE__ */ new Map();
function retainedClaimBytes(paths) {
const draining = drainingClaims.get(paths.file);
return listClaims(paths).reduce((n, c) => draining?.has(c.file) ? n : n + c.bytes, 0);
}
function heldBytes(paths) {
return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths);
}
function evictSent(paths, need) {
let freed = 0;
for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) {
Expand Down Expand Up @@ -713,7 +747,7 @@ function retainSent(url, eventId, body, mark, options = {}) {
const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS;
const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`);
return withLock(paths.lock, () => {
if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false;
if (fileSize(paths.file) + sentBytes(paths) + retainedClaimBytes(paths) + bytes > policy.maxBytes) return false;
const fd = openSync(file, "a", 384);
try {
writeSync(fd, line);
Expand Down Expand Up @@ -757,17 +791,7 @@ function reconcileSent(url, bootId, options = {}) {
return result;
}
function claimFiles(paths) {
const claimed = [];
try {
const prefix = `${paths.name}.draining-`;
for (const entry of readdirSync(paths.dir)) {
if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue;
const full = join(paths.dir, entry);
try {
if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full);
} catch {}
}
} catch {}
const claimed = eligibleClaims(paths).map((claim) => claim.file);
if (fileSize(paths.file) > 0) {
const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`);
withLock(paths.lock, (locked) => {
Expand Down Expand Up @@ -799,8 +823,12 @@ function requeueUnlocked(paths, leftover) {
utimesSync(target, 0, 0);
} catch {}
}
function eligibleClaims(paths, now = Date.now()) {
return listClaims(paths).filter((claim) => now - claim.mtimeMs > ORPHAN_CLAIM_MS);
}
function spoolHasRecords(url, dir) {
return fileSize(spoolPaths(url, dir).file) > 0;
const paths = spoolPaths(url, dir);
return fileSize(paths.file) > 0 || eligibleClaims(paths).length > 0;
}
function drainInProgress(url, dir) {
const paths = spoolPaths(url, dir);
Expand All @@ -819,6 +847,7 @@ async function drainSpool(url, send, options = {}) {
rejected: 0,
expired: 0,
corrupt: 0,
unreadable: 0,
remaining: 0
};
const paths = spoolPaths(url, options.dir);
Expand All @@ -840,12 +869,17 @@ async function drainSpool(url, send, options = {}) {
skipped: "empty"
};
const records = [];
const read = [];
for (const file of files) try {
const parsed = parseLines(readFileSync(file, "utf-8"));
records.push(...parsed.records);
result.corrupt += parsed.corrupt;
} catch {}
read.push(file);
} catch {
result.unreadable++;
}
result.claimed = records.length;
drainingClaims.set(paths.file, new Set(read));
const seen = /* @__PURE__ */ new Set();
const now = Date.now();
const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY;
Expand Down Expand Up @@ -888,7 +922,7 @@ async function drainSpool(url, send, options = {}) {
}
result.remaining = leftover.length;
withLock(paths.lock, (locked) => locked ? requeueLocked(paths, leftover) : requeueUnlocked(paths, leftover), 2e3);
for (const file of files) try {
for (const file of read) try {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
unlinkSync(file);
} catch {}
writeStats(paths, (s) => {
Expand All @@ -901,11 +935,13 @@ async function drainSpool(url, send, options = {}) {
s.lastDrainDuplicates = result.duplicates;
s.lastDrainRejected = result.rejected;
s.lastDrainRemaining = result.remaining;
s.lastDrainUnreadable = result.unreadable;
if (result.error) s.lastDrainError = result.error;
else delete s.lastDrainError;
});
return result;
} finally {
drainingClaims.delete(paths.file);
releaseLock(paths.drainLock);
}
}
Expand Down
114 changes: 79 additions & 35 deletions src/capture/spool.ts
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ export interface SpoolStats {
lastDrainDuplicates?: number;
lastDrainRejected?: number;
lastDrainRemaining?: number;
lastDrainUnreadable?: number;
lastDrainError?: string;
}

Expand All @@ -82,10 +83,10 @@ export interface SpoolSummary {
oldestAt: string | null;
retained: number;
retainedBytes: number;
unreadable: number;
maxBytes: number;
maxAgeHours: number;
stats: SpoolStats;
unreadable?: boolean;
}

export type SendOutcome = "delivered" | "duplicate" | "rejected" | "retry";
Expand All @@ -97,6 +98,7 @@ export interface DrainResult {
rejected: number;
expired: number;
corrupt: number;
unreadable: number;
remaining: number;
error?: string;
skipped?: "locked" | "empty" | "disabled";
Expand Down Expand Up @@ -339,10 +341,10 @@ export function appendSpool(
if (bytes > policy.maxRecordBytes) return note("too-large", bytes);
return withLock(paths.lock, (locked) => {
let expired = 0;
if (locked && fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy);
const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes;
if (locked && over > 0) evictSent(paths, over);
if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) {
if (locked && fileSize(paths.file) + claimBytes(paths) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy);
const over = heldBytes(paths) + bytes - policy.maxBytes;
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
if (locked && over > 0 && over <= sentBytes(paths)) evictSent(paths, over);
if (heldBytes(paths) + bytes > policy.maxBytes) {
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
if (expired) writeStats(paths, (s) => void (s.expired += expired));
return note("full", bytes);
}
Expand Down Expand Up @@ -396,6 +398,47 @@ function sentBytes(paths: SpoolPaths): number {
return listSent(paths).reduce((n, s) => n + s.bytes, 0);
}

interface ClaimFile {
file: string;
bytes: number;
mtimeMs: number;
}

function listClaims(paths: SpoolPaths): ClaimFile[] {
const claims: ClaimFile[] = [];
let entries: string[];
try {
entries = readdirSync(paths.dir);
} catch {
return claims;
}
const prefix = `${paths.name}.draining-`;
for (const entry of entries) {
if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue;
const file = join(paths.dir, entry);
try {
const stat = statSync(file);
claims.push({ file, bytes: stat.size, mtimeMs: stat.mtimeMs });
} catch {}
}
return claims;
}

function claimBytes(paths: SpoolPaths): number {
return listClaims(paths).reduce((n, c) => n + c.bytes, 0);
}

const drainingClaims = new Map<string, ReadonlySet<string>>();

function retainedClaimBytes(paths: SpoolPaths): number {
const draining = drainingClaims.get(paths.file);
return listClaims(paths).reduce((n, c) => (draining?.has(c.file) ? n : n + c.bytes), 0);
}

function heldBytes(paths: SpoolPaths): number {
return fileSize(paths.file) + sentBytes(paths) + claimBytes(paths);
}

function evictSent(paths: SpoolPaths, need: number): void {
let freed = 0;
for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) {
Expand Down Expand Up @@ -448,7 +491,7 @@ export function retainSent(
const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS;
const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`);
return withLock(paths.lock, () => {
if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false;
if (fileSize(paths.file) + sentBytes(paths) + retainedClaimBytes(paths) + bytes > policy.maxBytes) return false;
const fd = openSync(file, "a", 0o600);
try {
writeSync(fd, line);
Expand Down Expand Up @@ -496,17 +539,7 @@ export function reconcileSent(
}

function claimFiles(paths: SpoolPaths): string[] {
const claimed: string[] = [];
try {
const prefix = `${paths.name}.draining-`;
for (const entry of readdirSync(paths.dir)) {
if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue;
const full = join(paths.dir, entry);
try {
if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full);
} catch {}
}
} catch {}
const claimed = eligibleClaims(paths).map((claim) => claim.file);
if (fileSize(paths.file) > 0) {
const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`);
withLock(paths.lock, (locked) => {
Expand Down Expand Up @@ -541,8 +574,13 @@ function requeueUnlocked(paths: SpoolPaths, leftover: SpoolRecord[]): void {
} catch {}
}

function eligibleClaims(paths: SpoolPaths, now = Date.now()): ClaimFile[] {
return listClaims(paths).filter((claim) => now - claim.mtimeMs > ORPHAN_CLAIM_MS);
}

export function spoolHasRecords(url: string, dir?: string): boolean {
return fileSize(spoolPaths(url, dir).file) > 0;
const paths = spoolPaths(url, dir);
return fileSize(paths.file) > 0 || eligibleClaims(paths).length > 0;
}

export function drainInProgress(url: string, dir?: string): boolean {
Expand All @@ -560,7 +598,7 @@ export async function drainSpool(
options: { policy?: SpoolPolicy; dir?: string; maxRecords?: number; deadlineMs?: number } = {},
): Promise<DrainResult> {
const policy = options.policy ?? spoolPolicy();
const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 };
const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, unreadable: 0, remaining: 0 };
const paths = spoolPaths(url, options.dir);
if (!existsSync(paths.dir)) return { ...result, skipped: "empty" };
try {
Expand All @@ -571,14 +609,19 @@ export async function drainSpool(
const files = claimFiles(paths);
if (files.length === 0) return { ...result, skipped: "empty" };
const records: SpoolRecord[] = [];
const read: string[] = [];
for (const file of files) {
try {
const parsed = parseLines(readFileSync(file, "utf-8"));
records.push(...parsed.records);
result.corrupt += parsed.corrupt;
} catch {}
read.push(file);
} catch {
result.unreadable++;
}
}
result.claimed = records.length;
drainingClaims.set(paths.file, new Set(read));
const seen = new Set<string>();
const now = Date.now();
const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY;
Expand Down Expand Up @@ -618,7 +661,7 @@ export async function drainSpool(
}
result.remaining = leftover.length;
withLock(paths.lock, (locked) => (locked ? requeueLocked(paths, leftover) : requeueUnlocked(paths, leftover)), 2000);
for (const file of files) {
for (const file of read) {
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
try {
unlinkSync(file);
} catch {}
Expand All @@ -633,11 +676,13 @@ export async function drainSpool(
s.lastDrainDuplicates = result.duplicates;
s.lastDrainRejected = result.rejected;
s.lastDrainRemaining = result.remaining;
s.lastDrainUnreadable = result.unreadable;
if (result.error) s.lastDrainError = result.error;
else delete s.lastDrainError;
});
return result;
} finally {
drainingClaims.delete(paths.file);
releaseLock(paths.drainLock);
}
}
Expand All @@ -648,29 +693,28 @@ export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?:
let records = 0;
let bytes = 0;
let oldestAt: string | null = null;
const files: string[] = [];
let unreadable = false;
let unreadable = 0;
const noteError = (err: unknown) => {
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") unreadable = true;
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") unreadable++;
};
files.push(paths.file);
try {
for (const entry of readdirSync(paths.dir)) {
if (entry.startsWith(`${paths.name}.draining-`) && entry.endsWith(".jsonl")) files.push(join(paths.dir, entry));
}
readdirSync(paths.dir);
} catch (err) {
noteError(err);
}
const files = [paths.file, ...listClaims(paths).map((claim) => claim.file)];
for (const file of files) {
let text: string;
try {
const text = readFileSync(file, "utf-8");
bytes += Buffer.byteLength(text);
for (const rec of parseLines(text).records) {
records++;
if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt;
}
text = readFileSync(file, "utf-8");
} catch (err) {
noteError(err);
continue;
}
bytes += Buffer.byteLength(text);
for (const rec of parseLines(text).records) {
records++;
if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt;
}
}
let retained = 0;
Expand All @@ -690,9 +734,9 @@ export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?:
oldestAt,
retained,
retainedBytes,
unreadable,
maxBytes: policy.maxBytes,
maxAgeHours: Math.round(policy.maxAgeMs / 3600_000),
stats: readSpoolStats(paths),
...(unreadable ? { unreadable: true } : {}),
};
}
1 change: 1 addition & 0 deletions src/cli/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ export async function runCaptureCommand(options: CaptureCommandOptions): Promise
log(` limits: ${spool.maxBytes} bytes, ${spool.maxAgeHours} h`);
log(` totals: ${spool.stats.spooled} spooled, ${spool.stats.delivered} delivered, ${spool.stats.duplicates} duplicates, ${spool.stats.dropped} dropped, ${spool.stats.expired} expired`);
if (spool.stats.lastDrainAt) log(` last drain: ${spool.stats.lastDrainAt}`);
if (spool.unreadable) log(` unreadable spool files kept for retry: ${spool.unreadable}`);
if (spool.stats.lastDropAt) log(` last drop: ${spool.stats.lastDropAt} (${spool.stats.lastDropReason ?? "unknown"})`);

const server = output["server"] as {
Expand Down
Loading
Loading