diff --git a/plugins/codex-lcm/README.md b/plugins/codex-lcm/README.md index 5fe126a..8daa3f1 100644 --- a/plugins/codex-lcm/README.md +++ b/plugins/codex-lcm/README.md @@ -35,6 +35,16 @@ Storage defaults to `~/.codex-lcm`. Override storage for hook and MCP operations CODEX_LCM_HOME=/path/to/lcm-home node bin/codex-lcm health ``` +Raw history remains available forever unless `~/.codex-lcm/.env` sets a +positive retention window: + +```dotenv +CODEX_LCM_RETENTION_DAYS=90 +``` + +The process environment takes precedence over `.env`. Invalid, zero, negative, +and fractional values disable deletion and appear in `health` and `stats`. + Useful diagnostics and backfill commands: ```sh @@ -57,7 +67,7 @@ reasoning metadata, and cumulative token usage from existing transcripts. and cursor filters. `usage --roots-only` includes descendant session tokens, so root-task totals account for delegated work. `cleanup` previews a high-signal FTS rebuild and SQLite compaction; `--apply` performs it while preserving the -raw JSONL source of truth. `context-plan` +configured raw history. `context-plan` estimates recent-session token pressure and recommends when to pack LCM context; it does not control Codex compaction. The benchmark command generates a temporary synthetic long session and verifies old evidence diff --git a/plugins/codex-lcm/docs/architecture.md b/plugins/codex-lcm/docs/architecture.md index 7d8e852..13a6e7b 100644 --- a/plugins/codex-lcm/docs/architecture.md +++ b/plugins/codex-lcm/docs/architecture.md @@ -51,11 +51,26 @@ Default home: ```text ~/.codex-lcm/ - events.jsonl + events.jsonl # active append target, capped at 64 MiB + segments/ + manifest.json + *.jsonl.gz # verified closed segments index.sqlite ``` -`events.jsonl` is the source of truth. `index.sqlite` is derived and can be deleted or rebuilt later. +The active log and manifest-listed segments are the source of truth. +`index.sqlite` is derived and can be deleted or rebuilt later. Closed segments +use gzip level 1. SQLite keeps byte locators for archived events and drops its +duplicate JSON after the segment checksum and event IDs pass verification. + +Writable startup cuts an older single-file store over under the raw writer +lock. The hook gets a fresh active log at once, while a one-shot worker migrates +the renamed legacy file in bounded batches. The manifest records progress, so a +stopped worker resumes at the last published byte offset. + +The default retention policy is unlimited. A positive +`CODEX_LCM_RETENTION_DAYS` in `~/.codex-lcm/.env` expires closed raw segments, +detailed event rows, and orphan overflow files. Session and summary rows remain. SQLite tables: diff --git a/plugins/codex-lcm/docs/troubleshooting.md b/plugins/codex-lcm/docs/troubleshooting.md index 708f6cd..313ed6d 100644 --- a/plugins/codex-lcm/docs/troubleshooting.md +++ b/plugins/codex-lcm/docs/troubleshooting.md @@ -85,7 +85,8 @@ CODEX_LCM_HOME=/private/tmp/codex-lcm-check node bin/codex-lcm health --json CODEX_LCM_HOME=/private/tmp/codex-lcm-check node bin/codex-lcm stats --json ``` -If SQLite cannot open `index.sqlite`, Codex LCM still appends `events.jsonl` and falls back to raw-log scanning. +If SQLite cannot open `index.sqlite`, Codex LCM still appends `events.jsonl` and +falls back to scanning the active log and archived segments. Use `stats --json` when you need aggregate hook-event, summary-depth, graph-count, and freshness checks without opening the SQLite database directly. For compaction hook verification, check `hook_event_counts.PreCompact` and @@ -106,8 +107,8 @@ The preview reports the current index size, retained search rows, duplicated event-text bytes, and the projected high-signal search-row count. It does not write to SQLite. -`events.jsonl` remains the source of truth, so cleanup never deletes transcript -events. If you also want a point-in-time copy of the derived index, make an +The active log and archived segments remain the source of truth, so cleanup +never deletes retained transcript events. If you also want a point-in-time copy of the derived index, make an online SQLite backup first: ```sh @@ -127,6 +128,25 @@ duplicate event-text column, refreshes deterministic summaries, and runs first if SQLite reports that the database is busy. Set `CODEX_LCM_HOME` when cleaning a non-default LCM home. +## Retention and automatic migration + +Codex LCM keeps raw history forever by default. To set a limit, create +`~/.codex-lcm/.env` with one positive whole number: + +```dotenv +CODEX_LCM_RETENTION_DAYS=90 +``` + +A process-level value overrides the file. Check `config_error` in `health +--json` if deletion does not run. Finite retention removes exact old event +sources after the cutoff, but keeps session and summary records. + +An upgraded store migrates without a manual command. `migration_state` reports +`pending`, `complete`, or `error`; `plain_segment_count`, +`compressed_segment_count`, and `archive_bytes` show its progress. Do not remove +`segments/legacy.jsonl` when migration reports an error because it remains the +forensic source for any quarantined record. + ## Node Warnings The implementation uses Node 22's `node:sqlite`. Test and smoke scripts run with `--no-warnings` so experimental runtime warnings do not interfere with MCP stdout parsing. diff --git a/plugins/codex-lcm/src/cli.ts b/plugins/codex-lcm/src/cli.ts index a33f21d..4f6e26e 100644 --- a/plugins/codex-lcm/src/cli.ts +++ b/plugins/codex-lcm/src/cli.ts @@ -3,6 +3,7 @@ import { runLongContextBenchmark, runRetrievalQualityBenchmark } from "./benchma import { importCodexSessions } from "./codex-import.ts"; import { buildDoctorReport } from "./doctor.ts"; import { runHook } from "./hook.ts"; +import { runMaintenanceOnce } from "./maintenance.ts"; import { readStatus } from "./installer.ts"; import { startMcpServer } from "./mcp.ts"; import { createStorage } from "./storage.ts"; @@ -25,6 +26,10 @@ export async function main(argv: string[]): Promise { await runHook(rest); return; } + if (command === "maintain" && rest[0] === "--once") { + printObjectOrText(runMaintenanceOnce(loadConfig())); + return; + } if (command === "status") { printObjectOrText(readStatus({ codexHome: optionValue(rest, "--codex-home"), root: pluginRoot() })); return; diff --git a/plugins/codex-lcm/src/config.ts b/plugins/codex-lcm/src/config.ts index 5ff9ad0..59b5a72 100644 --- a/plugins/codex-lcm/src/config.ts +++ b/plugins/codex-lcm/src/config.ts @@ -1,3 +1,4 @@ +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -13,8 +14,13 @@ export type LcmLimits = { export type LcmConfig = { home: string; rawLogPath: string; + segmentsDir: string; + manifestPath: string; + maintenancePath: string; indexPath: string; overflowDir: string; + retentionDays?: number; + configError?: string; limits: LcmLimits; }; @@ -32,15 +38,47 @@ function resolveHome(env: Record = process.env): str export function loadConfig(options: { home?: string; env?: Record } = {}): LcmConfig { const home = path.resolve(options.home || resolveHome(options.env)); + const retention = retentionDays(home, options.env ?? process.env); + const segmentsDir = path.join(home, "segments"); return { home, rawLogPath: path.join(home, "events.jsonl"), + segmentsDir, + manifestPath: path.join(segmentsDir, "manifest.json"), + maintenancePath: path.join(home, "maintenance.lock.sqlite"), indexPath: path.join(home, "index.sqlite"), overflowDir: path.join(home, "overflow"), + retentionDays: retention.value, + configError: retention.error, limits: DEFAULT_LIMITS, }; } +function retentionDays(home: string, env: Record): { value?: number; error?: string } { + const raw = env.CODEX_LCM_RETENTION_DAYS ?? envFileValue(path.join(home, ".env")); + if (raw === undefined) return {}; + if (!/^[1-9][0-9]*$/u.test(raw)) { + return { error: "CODEX_LCM_RETENTION_DAYS must be a positive integer." }; + } + const value = Number(raw); + if (!Number.isSafeInteger(value)) { + return { error: "CODEX_LCM_RETENTION_DAYS must be a positive safe integer." }; + } + return { value }; +} + +function envFileValue(envPath: string): string | undefined { + if (!fs.existsSync(envPath)) return undefined; + let value: string | undefined; + for (const line of fs.readFileSync(envPath, "utf8").split(/\r?\n/u)) { + if (line.length === 0 || line.startsWith("#")) continue; + if (!line.startsWith("CODEX_LCM_RETENTION_DAYS=")) continue; + if (value !== undefined) return ""; + value = line.slice("CODEX_LCM_RETENTION_DAYS=".length); + } + return value; +} + export function pluginRoot(): string { return path.resolve(fileURLToPath(new URL("../", import.meta.url))); } diff --git a/plugins/codex-lcm/src/hook.ts b/plugins/codex-lcm/src/hook.ts index 7e03305..b900d7d 100644 --- a/plugins/codex-lcm/src/hook.ts +++ b/plugins/codex-lcm/src/hook.ts @@ -5,6 +5,7 @@ import { DEFAULT_LIMITS, loadConfig } from "./config.ts"; import { importCodexSessions } from "./codex-import.ts"; import { normalizeHookEvent } from "./events.ts"; import { resolveGitMetadata } from "./git.ts"; +import { queueMaintenance } from "./maintenance.ts"; import { sha256 } from "./redact.ts"; import { createStorage } from "./storage.ts"; @@ -71,6 +72,7 @@ export async function runHook(args: string[]): Promise { } finally { storage.close(); } + queueMaintenance(config); const output = postCompactRecoveryOutput({ home: config.home, hookEvent: event.hook_event, diff --git a/plugins/codex-lcm/src/maintenance.ts b/plugins/codex-lcm/src/maintenance.ts new file mode 100644 index 0000000..03a33b7 --- /dev/null +++ b/plugins/codex-lcm/src/maintenance.ts @@ -0,0 +1,586 @@ +import { createHash } from "node:crypto"; +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { gunzipSync, gzipSync } from "node:zlib"; + +import type { LcmConfig } from "./config.ts"; +import { parsePersistedEvent } from "./event-codec.ts"; +import type { NormalizedEvent } from "./events.ts"; +import { segmentedRawLogState, withRawLogLock } from "./raw-log.ts"; +import { emptySegmentManifest, readManifest, writeManifestAtomic, type SegmentRecord } from "./raw-segments.ts"; +import { + backfillLocatorMetadata, + clearVerifiedRawJson, + indexEventInTransaction, + invalidateRawLogState, + recordRawLogState, + segmentsNeedRawJsonClearing, +} from "./storage-persistence.ts"; +import { registerStoredEventReader } from "./stored-event.ts"; + +const LEGACY_RELATIVE_PATH = "segments/legacy.jsonl"; +const DEFAULT_SEGMENT_CAP_BYTES = 64 * 1024 * 1024; +const READ_BUFFER_BYTES = 64 * 1024; +const MAINTENANCE_MARKER = "CODEX_LCM_MAINTENANCE_WORKER"; + +export type MaintenanceReport = { + readonly migrated: number; + readonly compressed: number; + readonly expired: number; + readonly cleared: number; + readonly quarantined: number; + readonly errors: readonly string[]; +}; + +export type MaintenanceOptions = { + readonly maxSegments?: number; + readonly segmentCapBytes?: number; + readonly now?: () => Date; +}; + +type MigratedEvent = { + readonly event: NormalizedEvent; + readonly serialized: Buffer; + readonly offset: number; +}; + +type QuarantineRecord = { + readonly offset: number; + readonly length: number; + readonly sha256: string; +}; + +type MigrationBatch = { + readonly events: readonly MigratedEvent[]; + readonly quarantines: readonly QuarantineRecord[]; + readonly nextOffset: number; + readonly eof: boolean; +}; + +export function cutOverLegacyLog(config: LcmConfig): boolean { + return withRawLogLock(config.rawLogPath, () => { + if (fs.existsSync(config.manifestPath)) return false; + const legacyPath = path.join(config.home, LEGACY_RELATIVE_PATH); + if (fs.existsSync(legacyPath)) { + if (fs.existsSync(config.rawLogPath) && fs.statSync(config.rawLogPath).size > 0) { + throw new Error("Cannot recover legacy migration while the active raw log contains events."); + } + if (!fs.existsSync(config.rawLogPath)) fs.writeFileSync(config.rawLogPath, "", { mode: 0o600, flag: "wx" }); + writeManifestAtomic(config.manifestPath, { + version: 1, + migration: { legacy_path: LEGACY_RELATIVE_PATH, offset: 0, complete: false }, + segments: [], + }); + return true; + } + if (!fs.existsSync(config.rawLogPath) || fs.statSync(config.rawLogPath).size === 0) { + writeManifestAtomic(config.manifestPath, emptySegmentManifest()); + return false; + } + + fs.mkdirSync(config.segmentsDir, { recursive: true, mode: 0o700 }); + fs.renameSync(config.rawLogPath, legacyPath); + fsyncDirectory(config.segmentsDir); + fsyncDirectory(config.home); + try { + fs.writeFileSync(config.rawLogPath, "", { mode: 0o600, flag: "wx" }); + writeManifestAtomic(config.manifestPath, { + version: 1, + migration: { legacy_path: LEGACY_RELATIVE_PATH, offset: 0, complete: false }, + segments: [], + }); + return true; + } catch (error) { + if (fs.existsSync(config.rawLogPath) && fs.statSync(config.rawLogPath).size === 0) fs.unlinkSync(config.rawLogPath); + fs.renameSync(legacyPath, config.rawLogPath); + throw error; + } + }); +} + +export function migrationInProgress(config: LcmConfig): boolean { + if (!fs.existsSync(config.manifestPath)) return false; + return readManifest(config.manifestPath).migration?.complete === false; +} + +export function runMaintenanceOnce(config: LcmConfig, options: MaintenanceOptions = {}): MaintenanceReport { + const empty = { migrated: 0, compressed: 0, expired: 0, cleared: 0, quarantined: 0, errors: [] } satisfies MaintenanceReport; + if (!fs.existsSync(config.manifestPath)) return empty; + fs.mkdirSync(config.home, { recursive: true, mode: 0o700 }); + const coordinator = new DatabaseSync(config.maintenancePath, { timeout: 0 }); + let locked = false; + try { + fs.chmodSync(config.maintenancePath, 0o600); + try { + coordinator.exec("BEGIN IMMEDIATE"); + locked = true; + } catch (error) { + if (error instanceof Error && Reflect.get(error, "errcode") === 5) return empty; + throw error; + } + const migration = migrateLegacy(config, options); + if (migrationInProgress(config) || migration.errors.length > 0) return migration; + const segments = maintainSegments(config, options.now ?? (() => new Date())); + return { + migrated: migration.migrated, + compressed: segments.compressed, + expired: segments.expired, + cleared: segments.cleared, + quarantined: migration.quarantined, + errors: segments.errors, + }; + } finally { + if (locked) coordinator.exec("ROLLBACK"); + coordinator.close(); + } +} + +export function queueMaintenance(config: LcmConfig): void { + if (process.env[MAINTENANCE_MARKER] === "1" || !maintenanceNeeded(config)) return; + const entry = process.argv[1]; + if (!entry) return; + const child = spawn(process.execPath, ["--no-warnings", entry, "maintain", "--once"], { + detached: true, + stdio: "ignore", + env: { ...process.env, CODEX_LCM_HOME: config.home, [MAINTENANCE_MARKER]: "1" }, + }); + child.unref(); +} + +function maintenanceNeeded(config: LcmConfig): boolean { + if (!fs.existsSync(config.manifestPath)) return false; + const manifest = readManifest(config.manifestPath); + return manifest.migration?.complete === false + || manifest.segments.some((record) => !record.compressed) + || archivedPayloadClearingNeeded(config, manifest.segments.map((record) => record.id)) + || (config.retentionDays !== undefined && config.configError === undefined); +} + +function archivedPayloadClearingNeeded(config: LcmConfig, segmentIds: readonly string[]): boolean { + if (segmentIds.length === 0 || !fs.existsSync(config.indexPath)) return false; + let db: DatabaseSync | undefined; + try { + db = new DatabaseSync(config.indexPath, { readOnly: true }); + return segmentsNeedRawJsonClearing(db, segmentIds); + } catch (error) { + if (error instanceof Error) return true; + throw error; + } finally { + db?.close(); + } +} + +function migrateLegacy(config: LcmConfig, options: MaintenanceOptions): MaintenanceReport { + const initial = readManifest(config.manifestPath); + const migration = initial.migration; + if (!migration || migration.complete) { + return { migrated: 0, compressed: 0, expired: 0, cleared: 0, quarantined: 0, errors: [] }; + } + const legacyPath = path.join(config.home, migration.legacy_path); + if (!fs.existsSync(legacyPath)) { + return { migrated: 0, compressed: 0, expired: 0, cleared: 0, quarantined: 0, errors: ["Legacy migration source is missing."] }; + } + const segmentCapBytes = options.segmentCapBytes ?? DEFAULT_SEGMENT_CAP_BYTES; + if (!Number.isSafeInteger(segmentCapBytes) || segmentCapBytes <= 0) throw new TypeError("Segment cap must be a positive integer."); + const maxSegments = options.maxSegments ?? Number.POSITIVE_INFINITY; + if (!(maxSegments === Number.POSITIVE_INFINITY || (Number.isSafeInteger(maxSegments) && maxSegments > 0))) { + throw new TypeError("Maximum segment count must be a positive integer."); + } + + const db = fs.existsSync(config.indexPath) ? new DatabaseSync(config.indexPath, { timeout: 5_000 }) : undefined; + if (db) { + registerStoredEventReader(db, config); + backfillLocatorMetadata(db); + } + let migrated = 0; + let quarantined = quarantineCount(config); + let segmentsWritten = 0; + try { + for (;;) { + const current = readManifest(config.manifestPath); + const state = current.migration; + if (!state || state.complete || segmentsWritten >= maxSegments) break; + const batch = readMigrationBatch(legacyPath, state.offset, segmentCapBytes, db); + for (const record of batch.quarantines) writeQuarantine(config, record); + quarantined = quarantineCount(config); + if (batch.events.length > 0) { + const record = publishLegacySegment(config, current.segments, batch.events, batch.nextOffset, db); + migrated += record.event_count; + segmentsWritten += 1; + } else { + updateMigrationOffset(config, batch.nextOffset); + } + if (!batch.eof) continue; + const missingLocators = db + ? Number(db.prepare("SELECT COUNT(*) AS count FROM events WHERE segment_id IS NULL").get()?.count ?? 0) + : 0; + const errors = [ + ...(quarantined > 0 ? [`${quarantined} malformed legacy records were quarantined.`] : []), + ...(missingLocators > 0 ? [`${missingLocators} indexed events have no raw locator.`] : []), + ]; + finishMigration(config, batch.nextOffset, errors[0]); + if (errors.length === 0) fs.unlinkSync(legacyPath); + return { migrated, compressed: 0, expired: 0, cleared: 0, quarantined, errors }; + } + } finally { + db?.close(); + } + return { migrated, compressed: 0, expired: 0, cleared: 0, quarantined, errors: [] }; +} + +function maintainSegments(config: LcmConfig, now: () => Date): MaintenanceReport { + let cleared = 0; + let compressed = 0; + const errors: string[] = []; + const db = fs.existsSync(config.indexPath) ? new DatabaseSync(config.indexPath, { timeout: 5_000 }) : undefined; + if (db) { + registerStoredEventReader(db, config); + backfillLocatorMetadata(db); + } + try { + for (const record of readManifest(config.manifestPath).segments) { + if (db) cleared += clearVerifiedRawJson(db, record.id); + if (record.compressed) continue; + try { + compressSegment(config, record); + compressed += 1; + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } + const retention = expireSegments(config, db, now()); + errors.push(...retention.errors); + if (db && (cleared > 0 || retention.expired > 0)) { + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + db.exec("VACUUM"); + } + if (db && errors.length === 0) { + const state = withRawLogLock(config.rawLogPath, () => segmentedRawLogState(config)); + recordRawLogState(db, state); + } + return { migrated: 0, compressed, expired: retention.expired, cleared, quarantined: 0, errors }; + } finally { + db?.close(); + } +} + +function expireSegments( + config: LcmConfig, + db: DatabaseSync | undefined, + now: Date, +): { readonly expired: number; readonly errors: readonly string[] } { + if (config.configError) return { expired: 0, errors: [config.configError] }; + if (config.retentionDays === undefined) return { expired: 0, errors: [] }; + const cutoff = new Date(now.getTime() - config.retentionDays * 24 * 60 * 60 * 1_000).toISOString(); + const manifest = readManifest(config.manifestPath); + const expired = manifest.segments.filter((record) => record.last_timestamp < cutoff); + if (expired.length === 0) return { expired: 0, errors: [] }; + if (db) { + db.exec("BEGIN IMMEDIATE"); + try { + invalidateRawLogState(db); + for (const record of expired) { + db.prepare("DELETE FROM event_fts WHERE event_id IN (SELECT event_id FROM events WHERE segment_id = ?1)").run(record.id); + db.prepare("DELETE FROM file_refs WHERE observed_event_id IN (SELECT event_id FROM events WHERE segment_id = ?1)").run(record.id); + db.prepare("DELETE FROM events WHERE segment_id = ?1").run(record.id); + } + db.prepare(` + UPDATE sessions SET event_count = (SELECT COUNT(*) FROM events WHERE events.session_id = sessions.session_id) + `).run(); + db.exec("COMMIT"); + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + } + const expiredIds = new Set(expired.map((record) => record.id)); + withRawLogLock(config.rawLogPath, () => { + const current = readManifest(config.manifestPath); + writeManifestAtomic(config.manifestPath, { + ...current, + segments: current.segments.filter((record) => !expiredIds.has(record.id)), + }); + }); + for (const record of expired) { + const targetPath = path.join(config.home, record.path); + if (fs.existsSync(targetPath)) fs.unlinkSync(targetPath); + } + if (db) removeOrphanOverflow(config, db); + return { expired: expired.length, errors: [] }; +} + +function removeOrphanOverflow(config: LcmConfig, db: DatabaseSync): void { + if (!fs.existsSync(config.overflowDir)) return; + const live = new Set(db.prepare("SELECT DISTINCT overflow_sha256 FROM events WHERE overflow_sha256 IS NOT NULL").all() + .map((row) => row.overflow_sha256) + .filter((value): value is string => typeof value === "string")); + for (const name of fs.readdirSync(config.overflowDir)) { + const match = /^([a-f0-9]{64})\.json$/u.exec(name); + if (!match || live.has(match[1])) continue; + const targetPath = path.join(config.overflowDir, name); + if (fs.lstatSync(targetPath).isFile()) fs.unlinkSync(targetPath); + } +} + +function compressSegment(config: LcmConfig, record: SegmentRecord): void { + const plainPath = path.join(config.home, record.path); + const plain = fs.readFileSync(plainPath); + verifySegmentContent(plain, record); + const compressed = gzipSync(plain, { level: 1 }); + verifySegmentContent(gunzipSync(compressed), record); + const relativePath = `${record.path}.gz`; + const compressedPath = path.join(config.home, relativePath); + const temporaryPath = `${compressedPath}.tmp`; + const descriptor = fs.openSync(temporaryPath, "w", 0o600); + try { + fs.writeFileSync(descriptor, compressed); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fs.renameSync(temporaryPath, compressedPath); + withRawLogLock(config.rawLogPath, () => { + const manifest = readManifest(config.manifestPath); + writeManifestAtomic(config.manifestPath, { + ...manifest, + segments: manifest.segments.map((current) => current.id === record.id + ? { ...current, path: relativePath, compressed: true } + : current), + }); + }); + fs.unlinkSync(plainPath); +} + +function verifySegmentContent(content: Buffer, record: SegmentRecord): void { + if (hashBuffer(content) !== record.sha256) throw new Error(`Segment checksum failed: ${record.id}`); + const events = content.toString("utf8").split(/\r?\n/u).filter((line) => line.length > 0); + if (events.length !== record.event_count || events.some((line) => parsePersistedEvent(line) === undefined)) { + throw new Error(`Segment event count failed: ${record.id}`); + } +} + +function readMigrationBatch( + legacyPath: string, + startOffset: number, + segmentCapBytes: number, + db: DatabaseSync | undefined, +): MigrationBatch { + const descriptor = fs.openSync(legacyPath, "r"); + const readBuffer = Buffer.allocUnsafe(READ_BUFFER_BYTES); + const events: MigratedEvent[] = []; + const quarantines: QuarantineRecord[] = []; + let canonicalBytes = 0; + let readPosition = startOffset; + let pending = Buffer.alloc(0); + let pendingOffset = startOffset; + try { + for (;;) { + const bytesRead = fs.readSync(descriptor, readBuffer, 0, readBuffer.length, readPosition); + if (bytesRead === 0) { + if (pending.length > 0) { + const result = addLegacyLine(pending, pendingOffset, events, quarantines, canonicalBytes, segmentCapBytes, db); + if (!result.added) return { events, quarantines, nextOffset: pendingOffset, eof: false }; + canonicalBytes = result.canonicalBytes; + readPosition = pendingOffset + pending.length; + } + return { events, quarantines, nextOffset: readPosition, eof: true }; + } + const input = pending.length === 0 + ? Buffer.from(readBuffer.subarray(0, bytesRead)) + : Buffer.concat([pending, readBuffer.subarray(0, bytesRead)]); + const inputOffset = pendingOffset; + readPosition += bytesRead; + let lineStart = 0; + for (let index = 0; index < input.length; index += 1) { + if (input[index] !== 0x0a) continue; + const lineOffset = inputOffset + lineStart; + const line = input.subarray(lineStart, index); + const result = addLegacyLine(line, lineOffset, events, quarantines, canonicalBytes, segmentCapBytes, db, 1); + if (!result.added) return { events, quarantines, nextOffset: lineOffset, eof: false }; + canonicalBytes = result.canonicalBytes; + lineStart = index + 1; + if (canonicalBytes >= segmentCapBytes || inputOffset + lineStart - startOffset >= segmentCapBytes) { + return { events, quarantines, nextOffset: inputOffset + lineStart, eof: inputOffset + lineStart === fs.fstatSync(descriptor).size }; + } + } + pending = Buffer.from(input.subarray(lineStart)); + pendingOffset = inputOffset + lineStart; + } + } finally { + fs.closeSync(descriptor); + } +} + +function addLegacyLine( + line: Buffer, + offset: number, + events: MigratedEvent[], + quarantines: QuarantineRecord[], + canonicalBytes: number, + segmentCapBytes: number, + db: DatabaseSync | undefined, + newlineBytes = 0, +): { readonly added: boolean; readonly canonicalBytes: number } { + if (line.length === 0) return { added: true, canonicalBytes }; + const event = parsePersistedEvent(line.toString("utf8")) ?? repairMalformedEvent(line, db); + if (!event) { + quarantines.push({ offset, length: line.length + newlineBytes, sha256: hashBuffer(line) }); + return { added: true, canonicalBytes }; + } + const serialized = Buffer.from(`${JSON.stringify(event)}\n`, "utf8"); + if (events.length > 0 && canonicalBytes + serialized.length > segmentCapBytes) { + return { added: false, canonicalBytes }; + } + events.push({ event, serialized, offset: canonicalBytes }); + return { added: true, canonicalBytes: canonicalBytes + serialized.length }; +} + +function repairMalformedEvent(line: Buffer, db: DatabaseSync | undefined): NormalizedEvent | undefined { + if (!db) return undefined; + const match = /"event_id"\s*:\s*"([a-f0-9]{64})"/u.exec(line.subarray(0, 4_096).toString("utf8")); + const eventId = match?.[1]; + if (!eventId) return undefined; + const rawJson = db.prepare("SELECT raw_json FROM events WHERE event_id = ?1 AND raw_json <> ''").get(eventId)?.raw_json; + if (typeof rawJson !== "string") return undefined; + const event = parsePersistedEvent(rawJson); + return event?.event_id === eventId ? event : undefined; +} + +function publishLegacySegment( + config: LcmConfig, + existingSegments: readonly SegmentRecord[], + events: readonly MigratedEvent[], + nextOffset: number, + db: DatabaseSync | undefined, +): SegmentRecord { + const sequence = existingSegments.filter((segment) => segment.id.startsWith("legacy-")).length + 1; + const id = `legacy-${String(sequence).padStart(16, "0")}`; + const relativePath = `segments/${id}.jsonl`; + const segmentPath = path.join(config.home, relativePath); + const content = Buffer.concat(events.map((entry) => entry.serialized)); + const record: SegmentRecord = { + id, + path: relativePath, + compressed: false, + byte_count: content.length, + event_count: events.length, + first_timestamp: events[0]?.event.timestamp ?? "1970-01-01T00:00:00.000Z", + last_timestamp: events.at(-1)?.event.timestamp ?? "1970-01-01T00:00:00.000Z", + sha256: hashBuffer(content), + }; + writeSegmentFile(segmentPath, content, record.sha256); + if (db) { + db.exec("BEGIN IMMEDIATE"); + try { + const update = db.prepare(` + UPDATE events SET segment_id = ?1, raw_offset = ?2, raw_length = ?3 WHERE event_id = ?4 + `); + for (const entry of events) { + const location = { segmentId: id, offset: entry.offset, length: entry.serialized.length }; + const indexed = indexEventInTransaction(db, entry.event, false, location); + if (!indexed.inserted) update.run(id, entry.offset, entry.serialized.length, entry.event.event_id); + } + db.exec("COMMIT"); + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + } + withRawLogLock(config.rawLogPath, () => { + const manifest = readManifest(config.manifestPath); + const legacySegments = manifest.segments.filter((segment) => segment.id.startsWith("legacy-")); + const laterSegments = manifest.segments.filter((segment) => !segment.id.startsWith("legacy-")); + writeManifestAtomic(config.manifestPath, { + ...manifest, + migration: { legacy_path: manifest.migration?.legacy_path ?? LEGACY_RELATIVE_PATH, offset: nextOffset, complete: false }, + segments: [...legacySegments, record, ...laterSegments], + }); + }); + return record; +} + +function writeSegmentFile(segmentPath: string, content: Buffer, expectedHash: string): void { + if (fs.existsSync(segmentPath)) { + if (hashFile(segmentPath) !== expectedHash) throw new Error(`Existing migration segment failed verification: ${segmentPath}`); + return; + } + const temporaryPath = `${segmentPath}.tmp`; + const descriptor = fs.openSync(temporaryPath, "w", 0o600); + try { + fs.writeFileSync(descriptor, content); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fs.renameSync(temporaryPath, segmentPath); +} + +function updateMigrationOffset(config: LcmConfig, offset: number): void { + withRawLogLock(config.rawLogPath, () => { + const manifest = readManifest(config.manifestPath); + writeManifestAtomic(config.manifestPath, { + ...manifest, + migration: { legacy_path: manifest.migration?.legacy_path ?? LEGACY_RELATIVE_PATH, offset, complete: false }, + }); + }); +} + +function finishMigration(config: LcmConfig, offset: number, error: string | undefined): void { + withRawLogLock(config.rawLogPath, () => { + const manifest = readManifest(config.manifestPath); + writeManifestAtomic(config.manifestPath, { + ...manifest, + migration: { + legacy_path: manifest.migration?.legacy_path ?? LEGACY_RELATIVE_PATH, + offset, + complete: true, + ...(error ? { error } : {}), + }, + }); + }); +} + +function writeQuarantine(config: LcmConfig, record: QuarantineRecord): void { + const quarantineDir = path.join(config.segmentsDir, "quarantine"); + fs.mkdirSync(quarantineDir, { recursive: true, mode: 0o700 }); + const targetPath = path.join(quarantineDir, `${String(record.offset).padStart(20, "0")}.json`); + if (!fs.existsSync(targetPath)) fs.writeFileSync(targetPath, JSON.stringify(record), { mode: 0o600, flag: "wx" }); +} + +function quarantineCount(config: LcmConfig): number { + const quarantineDir = path.join(config.segmentsDir, "quarantine"); + if (!fs.existsSync(quarantineDir)) return 0; + return fs.readdirSync(quarantineDir).filter((name) => /^[0-9]{20}\.json$/u.test(name)).length; +} + +function hashBuffer(value: Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function hashFile(filePath: string): string { + const descriptor = fs.openSync(filePath, "r"); + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(READ_BUFFER_BYTES); + try { + for (;;) { + const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, null); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + } + } finally { + fs.closeSync(descriptor); + } + return hash.digest("hex"); +} + +function fsyncDirectory(directory: string): void { + if (process.platform === "win32") return; + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} diff --git a/plugins/codex-lcm/src/raw-log.ts b/plugins/codex-lcm/src/raw-log.ts index f5ce875..fc2eb90 100644 --- a/plugins/codex-lcm/src/raw-log.ts +++ b/plugins/codex-lcm/src/raw-log.ts @@ -1,9 +1,13 @@ import fs from "node:fs"; import path from "node:path"; +import { createHash } from "node:crypto"; import { DatabaseSync } from "node:sqlite"; +import { gunzipSync } from "node:zlib"; +import { loadConfig, type LcmConfig } from "./config.ts"; import { parsePersistedEvent } from "./event-codec.ts"; import type { NormalizedEvent } from "./events.ts"; +import { readManifest, segmentStoreState, writeManifestAtomic, type SegmentRecord } from "./raw-segments.ts"; export type RawLogReadResult = { readonly events: NormalizedEvent[]; @@ -14,10 +18,28 @@ export type RawLogState = { readonly size: number; readonly mtimeMs: number; readonly ctimeMs: number; + readonly segmentState?: string; +}; + +export type RawEventLocation = { + readonly segmentId: string; + readonly offset: number; + readonly length: number; +}; + +export type LocatedRawEvent = { + readonly event: NormalizedEvent; + readonly location?: RawEventLocation; +}; + +export type SegmentedAppendOptions = { + readonly segmentCapBytes?: number; }; const RAW_LOG_LOCK_TIMEOUT_MS = 10_000; const RAW_LOG_LOCK_POLL_MS = 10; +const DEFAULT_SEGMENT_CAP_BYTES = 64 * 1024 * 1024; +const RAW_READ_BUFFER_BYTES = 64 * 1024; const RAW_LOG_LOCK_WAIT = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); export class RawLogLockTimeoutError extends Error { @@ -88,6 +110,116 @@ export function appendRawEvents(rawLogPath: string, events: readonly NormalizedE } } +export function appendSegmentedEvents( + config: LcmConfig, + events: readonly NormalizedEvent[], + options: SegmentedAppendOptions = {}, +): RawEventLocation[] { + if (events.length === 0) return []; + const segmentCapBytes = options.segmentCapBytes ?? DEFAULT_SEGMENT_CAP_BYTES; + if (!Number.isSafeInteger(segmentCapBytes) || segmentCapBytes <= 0) { + throw new Error("Segment cap must be a positive integer."); + } + + const locations: RawEventLocation[] = []; + for (const event of events) { + const serialized = Buffer.from(`${JSON.stringify(event)}\n`, "utf8"); + const activeSize = fs.existsSync(config.rawLogPath) ? fs.statSync(config.rawLogPath).size : 0; + const separator = activeSize === 0 || !rawLogNeedsSeparator(config.rawLogPath) ? Buffer.alloc(0) : Buffer.from("\n"); + if (activeSize > 0 && activeSize + separator.length + serialized.length > segmentCapBytes) { + rotateActiveRawLog(config); + } + locations.push(appendActiveEvent(config.rawLogPath, activeSegmentId(config), serialized)); + } + return locations; +} + +export function* readAllRawEvents(config: LcmConfig): Generator { + for (const located of readAllLocatedRawEvents(config)) yield located.event; +} + +export function* readAllLocatedRawEvents(config: LcmConfig): Generator { + const manifest = readManifest(config.manifestPath); + if (manifest.migration?.complete === false) { + for (const event of readRawFileEvents(relativeStorePath(config, manifest.migration.legacy_path))) yield { event }; + yield* readActiveRawEvents(config); + return; + } + for (const record of manifest.segments) { + yield* readSegmentLocatedEvents(config, record); + } + yield* readActiveRawEvents(config); +} + +export function* readActiveRawEvents(config: LcmConfig): Generator { + yield* readRawFileLocatedEvents(config.rawLogPath, activeSegmentId(config)); +} + +export function readAllRawLog(config: LcmConfig): RawLogReadResult { + const events: NormalizedEvent[] = []; + let malformedLineCount = 0; + const manifest = readManifest(config.manifestPath); + if (manifest.migration?.complete === false) { + const legacy = readRawLog(relativeStorePath(config, manifest.migration.legacy_path)); + const active = readRawLog(config.rawLogPath); + return { + events: [...legacy.events, ...active.events], + malformedLineCount: legacy.malformedLineCount + active.malformedLineCount, + }; + } + for (const record of manifest.segments) { + const result = readSegmentLog(config, record); + events.push(...result.events); + malformedLineCount += result.malformedLineCount; + } + const active = readRawLog(config.rawLogPath); + events.push(...active.events); + return { events, malformedLineCount: malformedLineCount + active.malformedLineCount }; +} + +export function readLocatedEvent(config: LcmConfig, location: RawEventLocation): NormalizedEvent { + return createLocatedEventReader(config)(location); +} + +export function createLocatedEventReader(config: LcmConfig): (location: RawEventLocation) => NormalizedEvent { + let cachedSegmentId: string | undefined; + let cachedContent: Buffer | undefined; + return (location) => { + const record = readManifest(config.manifestPath).segments.find((segment) => segment.id === location.segmentId); + const targetPath = record + ? segmentPath(config, record) + : location.segmentId === activeSegmentId(config) ? config.rawLogPath : undefined; + if (!targetPath) throw new Error(`Unknown raw segment: ${location.segmentId}`); + if (!Number.isSafeInteger(location.offset) || location.offset < 0 || !Number.isSafeInteger(location.length) || location.length <= 0) { + throw new Error("Invalid raw event location."); + } + if (record?.compressed) { + if (cachedSegmentId !== record.id) { + cachedContent = gunzipSync(fs.readFileSync(targetPath)); + cachedSegmentId = record.id; + } + const content = cachedContent; + if (!content) throw new Error(`Compressed raw segment could not be read: ${record.id}`); + const serialized = content.subarray(location.offset, location.offset + location.length); + if (serialized.length !== location.length) throw new Error("Raw event location is outside the segment."); + const event = parsePersistedEvent(serialized.toString("utf8").trim()); + if (!event) throw new Error("Raw event location does not contain a persisted event."); + return event; + } + const descriptor = fs.openSync(targetPath, "r"); + try { + const serialized = Buffer.allocUnsafe(location.length); + const bytesRead = fs.readSync(descriptor, serialized, 0, serialized.length, location.offset); + if (bytesRead !== serialized.length) throw new Error("Raw event location is outside the segment."); + const event = parsePersistedEvent(serialized.toString("utf8").trim()); + if (!event) throw new Error("Raw event location does not contain a persisted event."); + return event; + } finally { + fs.closeSync(descriptor); + } + }; +} + function restoreRawLog(rawLogPath: string, existed: boolean, previousSize: number): void { if (existed) { fs.truncateSync(rawLogPath, previousSize); @@ -124,7 +256,7 @@ export function readRawLog(rawLogPath: string): RawLogReadResult { if (!fs.existsSync(rawLogPath)) return { events: [], malformedLineCount: 0 }; const events: NormalizedEvent[] = []; let malformedLineCount = 0; - for (const line of fs.readFileSync(rawLogPath, "utf8").split(/\r?\n/u)) { + for (const line of readRawFileLines(rawLogPath)) { if (line.trim().length === 0) continue; const event = parsePersistedEvent(line); if (event) events.push(event); @@ -134,6 +266,10 @@ export function readRawLog(rawLogPath: string): RawLogReadResult { } export function readRawEvents(rawLogPath: string): NormalizedEvent[] { + const config = loadConfig({ home: path.dirname(rawLogPath) }); + if (rawLogPath === config.rawLogPath && fs.existsSync(config.manifestPath)) { + return Array.from(readAllRawEvents(config)); + } return readRawLog(rawLogPath).events; } @@ -151,3 +287,232 @@ export function rawLogState(rawLogPath: string): RawLogState { ? { size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs } : { size: 0, mtimeMs: 0, ctimeMs: 0 }; } + +export function segmentedRawLogState(config: LcmConfig): RawLogState { + return { + ...rawLogState(config.rawLogPath), + segmentState: segmentStoreState(readManifest(config.manifestPath)), + }; +} + +function appendActiveEvent(rawLogPath: string, segmentId: string, serialized: Buffer): RawEventLocation { + fs.mkdirSync(path.dirname(rawLogPath), { recursive: true, mode: 0o700 }); + const existed = fs.existsSync(rawLogPath); + const previousSize = existed ? fs.statSync(rawLogPath).size : 0; + const separator = previousSize === 0 || !rawLogNeedsSeparator(rawLogPath) ? Buffer.alloc(0) : Buffer.from("\n"); + try { + fs.appendFileSync(rawLogPath, Buffer.concat([separator, serialized]), { mode: 0o600 }); + fsyncPath(rawLogPath, true); + if (!existed && process.platform !== "win32") fsyncPath(path.dirname(rawLogPath)); + } catch (error) { + try { + restoreRawLog(rawLogPath, existed, previousSize); + } catch (rollbackError) { + throw new AggregateError([error, rollbackError], "Raw log append failed and rollback failed."); + } + throw error; + } + return { segmentId, offset: previousSize + separator.length, length: serialized.length }; +} + +function rotateActiveRawLog(config: LcmConfig): void { + if (!fs.existsSync(config.rawLogPath) || fs.statSync(config.rawLogPath).size === 0) return; + const manifest = readManifest(config.manifestPath); + const id = nextSegmentId(manifest.segments); + const relativePath = path.join("segments", `${id}.jsonl`); + const destinationPath = path.join(config.home, relativePath); + if (fs.existsSync(destinationPath)) throw new Error(`Raw segment already exists: ${id}`); + const summary = summarizeRawFile(config.rawLogPath); + fs.mkdirSync(config.segmentsDir, { recursive: true, mode: 0o700 }); + fsyncPath(config.rawLogPath, true); + fs.renameSync(config.rawLogPath, destinationPath); + try { + writeManifestAtomic(config.manifestPath, { + ...manifest, + segments: [...manifest.segments, { + id, + path: relativePath, + compressed: false, + byte_count: summary.byteCount, + event_count: summary.eventCount, + first_timestamp: summary.firstTimestamp, + last_timestamp: summary.lastTimestamp, + sha256: summary.sha256, + }], + }); + if (process.platform !== "win32") fsyncPath(config.home); + } catch (error) { + fs.renameSync(destinationPath, config.rawLogPath); + throw error; + } +} + +function nextSegmentId(segments: readonly SegmentRecord[]): string { + const lastNumericId = segments.reduce((maximum, segment) => { + const value = /^[0-9]+$/u.test(segment.id) ? Number(segment.id) : 0; + return Number.isSafeInteger(value) ? Math.max(maximum, value) : maximum; + }, 0); + return String(lastNumericId + 1).padStart(16, "0"); +} + +function activeSegmentId(config: LcmConfig): string { + return nextSegmentId(readManifest(config.manifestPath).segments); +} + +function segmentPath(config: LcmConfig, record: SegmentRecord): string { + const targetPath = path.resolve(config.home, record.path); + const relative = path.relative(config.home, targetPath); + if (relative === "" || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`Segment path escapes the LCM home: ${record.id}`); + } + return targetPath; +} + +function* readSegmentLocatedEvents(config: LcmConfig, record: SegmentRecord): Generator { + if (!record.compressed) { + yield* readRawFileLocatedEvents(segmentPath(config, record), record.id); + return; + } + yield* readRawBufferLocatedEvents(gunzipSync(fs.readFileSync(segmentPath(config, record))), record.id); +} + +function readSegmentLog(config: LcmConfig, record: SegmentRecord): RawLogReadResult { + return record.compressed + ? parseRawBuffer(gunzipSync(fs.readFileSync(segmentPath(config, record)))) + : readRawLog(segmentPath(config, record)); +} + +function parseRawBuffer(content: Buffer): RawLogReadResult { + const events: NormalizedEvent[] = []; + let malformedLineCount = 0; + for (const line of content.toString("utf8").split(/\r?\n/u)) { + if (line.trim().length === 0) continue; + const event = parsePersistedEvent(line); + if (event) events.push(event); + else malformedLineCount += 1; + } + return { events, malformedLineCount }; +} + +function relativeStorePath(config: LcmConfig, relativePath: string): string { + const targetPath = path.resolve(config.home, relativePath); + const relative = path.relative(config.home, targetPath); + if (relative === "" || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error("Raw store path escapes the LCM home."); + } + return targetPath; +} + +function summarizeRawFile(rawLogPath: string): { + readonly byteCount: number; + readonly eventCount: number; + readonly firstTimestamp: string; + readonly lastTimestamp: string; + readonly sha256: string; +} { + const stat = fs.statSync(rawLogPath); + const events = readRawLog(rawLogPath).events; + const first = events[0]; + const last = events.at(-1); + return { + byteCount: stat.size, + eventCount: events.length, + firstTimestamp: first?.timestamp ?? "1970-01-01T00:00:00.000Z", + lastTimestamp: last?.timestamp ?? "1970-01-01T00:00:00.000Z", + sha256: hashRawFile(rawLogPath), + }; +} + +function hashRawFile(rawLogPath: string): string { + const descriptor = fs.openSync(rawLogPath, "r"); + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(RAW_READ_BUFFER_BYTES); + try { + for (let offset = 0; ; offset += RAW_READ_BUFFER_BYTES) { + const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, offset); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + } + } finally { + fs.closeSync(descriptor); + } + return hash.digest("hex"); +} + +function* readRawFileEvents(rawLogPath: string): Generator { + if (!fs.existsSync(rawLogPath)) return; + for (const line of readRawFileLines(rawLogPath)) { + if (line.trim().length === 0) continue; + const event = parsePersistedEvent(line); + if (event) yield event; + } +} + +function* readRawFileLocatedEvents(rawLogPath: string, segmentId: string): Generator { + if (!fs.existsSync(rawLogPath)) return; + for (const record of readRawFileLineRecords(rawLogPath)) { + if (record.line.trim().length === 0) continue; + const event = parsePersistedEvent(record.line); + if (event) { + yield { event, location: { segmentId, offset: record.offset, length: record.length } }; + } + } +} + +function* readRawBufferLocatedEvents(content: Buffer, segmentId: string): Generator { + let lineStart = 0; + for (let index = 0; index <= content.length; index += 1) { + if (index < content.length && content[index] !== 0x0a) continue; + const length = index - lineStart + (index < content.length ? 1 : 0); + const line = content.subarray(lineStart, index).toString("utf8").replace(/\r$/u, ""); + if (line.trim().length > 0) { + const event = parsePersistedEvent(line); + if (event) yield { event, location: { segmentId, offset: lineStart, length } }; + } + lineStart = index + 1; + } +} + +function* readRawFileLines(rawLogPath: string): Generator { + for (const record of readRawFileLineRecords(rawLogPath)) yield record.line; +} + +function* readRawFileLineRecords( + rawLogPath: string, +): Generator<{ readonly line: string; readonly offset: number; readonly length: number }> { + const descriptor = fs.openSync(rawLogPath, "r"); + const buffer = Buffer.allocUnsafe(RAW_READ_BUFFER_BYTES); + let remainder = Buffer.alloc(0); + let remainderOffset = 0; + try { + for (let offset = 0; ; offset += RAW_READ_BUFFER_BYTES) { + const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, offset); + if (bytesRead === 0) break; + const input = remainder.length === 0 + ? Buffer.from(buffer.subarray(0, bytesRead)) + : Buffer.concat([remainder, buffer.subarray(0, bytesRead)]); + let lineStart = 0; + for (let index = 0; index < input.length; index += 1) { + if (input[index] !== 0x0a) continue; + const line = input.subarray(lineStart, index); + yield { + line: line.toString("utf8").replace(/\r$/u, ""), + offset: remainderOffset + lineStart, + length: index - lineStart + 1, + }; + lineStart = index + 1; + } + remainder = Buffer.from(input.subarray(lineStart)); + remainderOffset += lineStart; + } + if (remainder.length > 0) { + yield { + line: remainder.toString("utf8").replace(/\r$/u, ""), + offset: remainderOffset, + length: remainder.length, + }; + } + } finally { + fs.closeSync(descriptor); + } +} diff --git a/plugins/codex-lcm/src/raw-segments.ts b/plugins/codex-lcm/src/raw-segments.ts new file mode 100644 index 0000000..2be4dfa --- /dev/null +++ b/plugins/codex-lcm/src/raw-segments.ts @@ -0,0 +1,167 @@ +import fs from "node:fs"; +import path from "node:path"; + +import type { LcmConfig } from "./config.ts"; +import { sha256 } from "./redact.ts"; + +export type SegmentRecord = { + id: string; + path: string; + compressed: boolean; + byte_count: number; + event_count: number; + first_timestamp: string; + last_timestamp: string; + sha256: string; +}; + +export type SegmentManifest = { + version: 1; + migration?: { legacy_path: string; offset: number; complete: boolean; error?: string }; + segments: SegmentRecord[]; +}; + +export type SegmentStorageHealth = { + readonly storage_layout: "segmented-v1"; + readonly migration_state: "none" | "pending" | "complete" | "error"; + readonly active_bytes: number; + readonly archive_bytes: number; + readonly plain_segment_count: number; + readonly compressed_segment_count: number; + readonly config_error?: string; +}; + +export function emptySegmentManifest(): SegmentManifest { + return { version: 1, segments: [] }; +} + +export function readManifest(manifestPath: string): SegmentManifest { + if (!fs.existsSync(manifestPath)) return emptySegmentManifest(); + try { + return validateManifest(JSON.parse(fs.readFileSync(manifestPath, "utf8"))); + } catch (error) { + throw new Error("Invalid segment manifest.", { cause: error }); + } +} + +export function writeManifestAtomic(manifestPath: string, manifest: SegmentManifest): void { + const serialized = JSON.stringify(validateManifest(manifest)); + const directory = path.dirname(manifestPath); + const temporaryPath = `${manifestPath}.tmp`; + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const descriptor = fs.openSync(temporaryPath, "w", 0o600); + try { + fs.writeFileSync(descriptor, serialized, "utf8"); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fs.renameSync(temporaryPath, manifestPath); + fsyncDirectory(directory); +} + +export function segmentStoreState(manifest: SegmentManifest): string { + return sha256(JSON.stringify(manifest)); +} + +export function segmentStorageHealth(config: LcmConfig): SegmentStorageHealth { + const manifestExists = fs.existsSync(config.manifestPath); + const manifest = readManifest(config.manifestPath); + const migration = manifest.migration; + const migrationState = !manifestExists || !migration + ? "none" + : migration.error ? "error" : migration.complete ? "complete" : "pending"; + return { + storage_layout: "segmented-v1", + migration_state: migrationState, + active_bytes: fs.existsSync(config.rawLogPath) ? fs.statSync(config.rawLogPath).size : 0, + archive_bytes: manifest.segments.reduce((total, record) => { + const segmentPath = path.join(config.home, record.path); + return total + (fs.existsSync(segmentPath) ? fs.statSync(segmentPath).size : 0); + }, 0), + plain_segment_count: manifest.segments.filter((record) => !record.compressed).length, + compressed_segment_count: manifest.segments.filter((record) => record.compressed).length, + ...(config.configError ? { config_error: config.configError } : {}), + }; +} + +function validateManifest(value: unknown): SegmentManifest { + if (!isRecord(value) || value.version !== 1 || !Array.isArray(value.segments)) fail(); + const migration = value.migration === undefined ? undefined : validateMigration(value.migration); + return { + version: 1, + ...(migration === undefined ? {} : { migration }), + segments: value.segments.map(validateSegment), + }; +} + +function validateMigration(value: unknown): NonNullable { + if (!isRecord(value) || !isRelativePath(value.legacy_path) || !isNonNegativeInteger(value.offset) || typeof value.complete !== "boolean") { + fail(); + } + if (value.error !== undefined && !isNonEmptyString(value.error)) fail(); + return { + legacy_path: value.legacy_path, + offset: value.offset, + complete: value.complete, + ...(value.error === undefined ? {} : { error: value.error }), + }; +} + +function validateSegment(value: unknown): SegmentRecord { + if (!isRecord(value) + || !isNonEmptyString(value.id) + || !isRelativePath(value.path) + || typeof value.compressed !== "boolean" + || !isNonNegativeInteger(value.byte_count) + || !isNonNegativeInteger(value.event_count) + || !isNonEmptyString(value.first_timestamp) + || !isNonEmptyString(value.last_timestamp) + || !isNonEmptyString(value.sha256) + || !/^[a-f0-9]{64}$/iu.test(value.sha256)) { + fail(); + } + return { + id: value.id, + path: value.path, + compressed: value.compressed, + byte_count: value.byte_count, + event_count: value.event_count, + first_timestamp: value.first_timestamp, + last_timestamp: value.last_timestamp, + sha256: value.sha256, + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function isRelativePath(value: unknown): value is string { + return isNonEmptyString(value) + && !path.isAbsolute(value) + && !path.win32.isAbsolute(value) + && !value.split(/[\\/]+/u).includes(".."); +} + +function fail(): never { + throw new Error("Malformed segment manifest."); +} + +function fsyncDirectory(directory: string): void { + if (process.platform === "win32") return; + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} diff --git a/plugins/codex-lcm/src/storage-context.ts b/plugins/codex-lcm/src/storage-context.ts index 8ab4b45..874233e 100644 --- a/plugins/codex-lcm/src/storage-context.ts +++ b/plugins/codex-lcm/src/storage-context.ts @@ -6,6 +6,7 @@ import type { FileReference } from "./file-refs.ts"; import { overflowReferenceFromEvent, readOverflowContent, type OverflowReference } from "./overflow.ts"; import { readRawEvents } from "./raw-log.ts"; import { recordValue, rowToFileReference } from "./storage-rows.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import { clampLimit, positiveInteger, @@ -190,7 +191,7 @@ export function getRecentContext( } const rows = db.prepare(` SELECT raw_json FROM ( - SELECT raw_json, timestamp, rowid + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json, timestamp, rowid FROM events WHERE session_id = ?1 ORDER BY timestamp DESC, rowid DESC @@ -289,9 +290,9 @@ export function getOverflowRef(db: DatabaseSync | undefined, rawLogPath: string, .find((reference) => reference?.sha256 === hash); } const rawJson = recordValue(db.prepare(` - SELECT raw_json + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events - WHERE json_extract(raw_json, '$.payload.overflow_ref.sha256') = ?1 + WHERE overflow_sha256 = ?1 ORDER BY timestamp DESC, rowid DESC LIMIT 1 `).get(hash)).raw_json; @@ -599,7 +600,7 @@ function getContextPlanEvents(db: DatabaseSync | undefined, rawLogPath: string, } const rows = db.prepare(` SELECT raw_json FROM ( - SELECT raw_json, timestamp, rowid + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json, timestamp, rowid FROM events WHERE session_id = ?1 ORDER BY timestamp DESC, rowid DESC diff --git a/plugins/codex-lcm/src/storage-graph.ts b/plugins/codex-lcm/src/storage-graph.ts index 2014fef..2d6140c 100644 --- a/plugins/codex-lcm/src/storage-graph.ts +++ b/plugins/codex-lcm/src/storage-graph.ts @@ -4,6 +4,7 @@ import { decodePersistedEvent } from "./event-codec.ts"; import type { NormalizedEvent } from "./events.ts"; import { readRawEvents } from "./raw-log.ts"; import { recordValue } from "./storage-rows.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import { countMap, extractEventMetadata } from "./storage-sessions.ts"; import { getSummaryNodesForGraph } from "./storage-summaries.ts"; import type { GraphEdge, GraphNode, SessionGraph } from "./storage-types.ts"; @@ -215,7 +216,7 @@ export function getStoredSessionGraph( : Math.max(0, Math.floor(limit / 4)); const graphNodeLimit = Math.max(1, limit - summaryBudget); const events = db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 ORDER BY timestamp ASC, rowid ASC LIMIT ?2 @@ -249,7 +250,7 @@ export function getLatestCheckpoint(db: DatabaseSync | undefined, sessionId: str if (!db) return undefined; const row = recordValue(db.prepare(` SELECT raw_json, position FROM ( - SELECT raw_json, hook_event, + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json, hook_event, ROW_NUMBER() OVER (ORDER BY timestamp, rowid) AS position FROM events WHERE session_id = ?1 diff --git a/plugins/codex-lcm/src/storage-pack.ts b/plugins/codex-lcm/src/storage-pack.ts index 8b5deec..b968426 100644 --- a/plugins/codex-lcm/src/storage-pack.ts +++ b/plugins/codex-lcm/src/storage-pack.ts @@ -5,6 +5,7 @@ import type { NormalizedEvent } from "./events.ts"; import { readRawEvents } from "./raw-log.ts"; import { getLatestCheckpoint } from "./storage-graph.ts"; import { recordValue } from "./storage-rows.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import { bestMatchSnippet, compactWhitespace, @@ -108,7 +109,7 @@ function searchContextEvents(db: DatabaseSync | undefined, rawLogPath: string, a : ""; const limitParameter = sessionIds.length + 3; const statement = db.prepare(` - SELECT e.raw_json + SELECT lcm_raw_json(e.raw_json, e.segment_id, e.raw_offset, e.raw_length) AS raw_json FROM event_fts f JOIN events e ON e.event_id = f.event_id WHERE event_fts MATCH ?1 @@ -140,7 +141,7 @@ function getRecentContextEvents(db: DatabaseSync | undefined, rawLogPath: string .reverse(); } return db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 AND hook_event IN ${SUMMARY_SOURCE_HOOKS} ORDER BY timestamp DESC, rowid DESC diff --git a/plugins/codex-lcm/src/storage-persistence.ts b/plugins/codex-lcm/src/storage-persistence.ts index 2a1dcda..4612e6c 100644 --- a/plugins/codex-lcm/src/storage-persistence.ts +++ b/plugins/codex-lcm/src/storage-persistence.ts @@ -5,10 +5,13 @@ import type { LcmConfig } from "./config.ts"; import { decodePersistedEvent } from "./event-codec.ts"; import type { NormalizedEvent } from "./events.ts"; import { extractFileReferences } from "./file-refs.ts"; -import { rawLogState, rawLogStat, readRawEventIds, readRawEvents, type RawLogState } from "./raw-log.ts"; +import { overflowReferenceFromEvent } from "./overflow.ts"; +import { rawLogState, rawLogStat, readRawEventIds, readRawEvents, segmentedRawLogState, type RawEventLocation, type RawLogState } from "./raw-log.ts"; import { eventSearchText } from "./storage-context.ts"; import { recordValue } from "./storage-rows.ts"; import { initializeStorageSchema } from "./storage-schema.ts"; +import { segmentStorageHealth } from "./raw-segments.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import { getSummaryBackfillSessionIds, rebuildSessionMemorySummary, shouldRebuildSessionMemorySummary } from "./storage-summaries.ts"; import { extractEventMetadata, extractSessionMetadata, isCodexLcmToolEvent, isSearchIndexEvent, maxNullable, scalar, summarizeSessions } from "./storage-sessions.ts"; import type { Health, IndexCleanupReport } from "./storage-types.ts"; @@ -18,6 +21,7 @@ const SUMMARY_SOURCE_HOOKS = "('UserPromptSubmit', 'Note', 'Stop', 'PreCompact', const FILE_REF_BACKFILL_KEY = "file_refs_backfilled_v1"; const DELEGATION_PARENT_BACKFILL_KEY = "delegation_parent_backfilled_v1"; const EVENT_METADATA_BACKFILL_KEY = "event_metadata_backfilled_v1"; +const EVENT_LOCATOR_METADATA_BACKFILL_KEY = "event_locator_metadata_backfilled_v1"; const RAW_LOG_INDEX_STATE_KEY = "raw_log_index_state_v1"; export type IndexEventResult = { readonly inserted: boolean; readonly summaryTouched: boolean }; @@ -88,7 +92,7 @@ export function emptyCleanupReport(indexPath: string): IndexCleanupReport { export function inspectIndexForCleanup(db: DatabaseSync, indexPath: string): CleanupInspection { const searchableEvents = db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE hook_event IN ${SUMMARY_SOURCE_HOOKS} ORDER BY timestamp ASC, rowid ASC `).all() @@ -158,6 +162,7 @@ export function writableIndexHealth( graphEdgeCounts: Record, ): Health { return { + ...segmentStorageHealth(config), home: config.home, raw_log_path: config.rawLogPath, index_path: config.indexPath, raw_log_exists: fs.existsSync(config.rawLogPath), index_exists: fs.existsSync(config.indexPath), index_available: true, ...(indexError ? { index_error: indexError } : {}), @@ -172,6 +177,7 @@ export function writableIndexHealth( export function rawHealth(config: LcmConfig, indexError: string | undefined): Health { const rawEvents = readRawEvents(config.rawLogPath); return { + ...segmentStorageHealth(config), home: config.home, raw_log_path: config.rawLogPath, index_path: config.indexPath, raw_log_exists: fs.existsSync(config.rawLogPath), index_exists: fs.existsSync(config.indexPath), index_available: false, ...(indexError ? { index_error: indexError } : {}), @@ -209,7 +215,7 @@ export function knownEventIds(db: DatabaseSync | undefined, rawLogPath: string, } export function indexedEventsById(db: DatabaseSync): Map { - return new Map(db.prepare("SELECT event_id, raw_json FROM events").all().map((row) => { + return new Map(db.prepare(`SELECT event_id, ${STORED_EVENT_JSON_SQL} AS raw_json FROM events`).all().map((row) => { const record = recordValue(row); return [String(record.event_id), String(record.raw_json)]; })); @@ -220,8 +226,31 @@ export function indexedRawLogState(db: DatabaseSync): string | undefined { return typeof row.value === "string" ? row.value : undefined; } -export function isRawLogIndexed(db: DatabaseSync, rawLogPath: string): boolean { - return indexedRawLogState(db) === JSON.stringify(rawLogState(rawLogPath)); +export function indexedActiveLogIsAppendOnly(db: DatabaseSync, config: LcmConfig): boolean { + const parsed = parsedIndexedRawLogState(db); + const current = segmentedRawLogState(config); + return parsed !== undefined && parsed.segmentState === current.segmentState && current.size > parsed.size; +} + +function parsedIndexedRawLogState(db: DatabaseSync): { readonly size: number; readonly segmentState?: string } | undefined { + const state = indexedRawLogState(db); + if (!state) return undefined; + try { + const parsed = JSON.parse(state); + if (typeof parsed !== "object" || parsed === null || typeof Reflect.get(parsed, "size") !== "number") return undefined; + const segmentState = Reflect.get(parsed, "segmentState"); + return { + size: Number(Reflect.get(parsed, "size")), + ...(typeof segmentState === "string" ? { segmentState } : {}), + }; + } catch (error) { + if (error instanceof SyntaxError) return undefined; + throw error; + } +} + +export function isRawLogIndexed(db: DatabaseSync, config: LcmConfig): boolean { + return indexedRawLogState(db) === JSON.stringify(segmentedRawLogState(config)); } export function recordRawLogState(db: DatabaseSync, state: RawLogState): void { @@ -231,8 +260,12 @@ export function recordRawLogState(db: DatabaseSync, state: RawLogState): void { `).run(RAW_LOG_INDEX_STATE_KEY, JSON.stringify(state)); } -export function currentRawLogState(rawLogPath: string): RawLogState { - return rawLogState(rawLogPath); +export function invalidateRawLogState(db: DatabaseSync): void { + db.prepare("DELETE FROM index_metadata WHERE key = ?1").run(RAW_LOG_INDEX_STATE_KEY); +} + +export function currentRawLogState(config: LcmConfig): RawLogState { + return segmentedRawLogState(config); } export function initializeIndex(db: DatabaseSync): void { @@ -241,18 +274,26 @@ export function initializeIndex(db: DatabaseSync): void { if (backfillSessionMetadata) backfillExistingSessionMetadata(db); } -export function indexEventInTransaction(db: DatabaseSync | undefined, event: NormalizedEvent, rebuildSummary: boolean): IndexEventResult { +export function indexEventInTransaction( + db: DatabaseSync | undefined, + event: NormalizedEvent, + rebuildSummary: boolean, + location?: RawEventLocation, +): IndexEventResult { if (!db) return { inserted: false, summaryTouched: false }; const metadata = extractEventMetadata(event); const sessionMetadata = extractSessionMetadata(event); const insert = db.prepare(` INSERT OR IGNORE INTO events - (event_id, session_id, timestamp, hook_event, cwd, repo_root, git_branch, turn_id, tool_use_id, text, raw_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + (event_id, session_id, timestamp, hook_event, cwd, repo_root, git_branch, turn_id, tool_use_id, text, raw_json, + segment_id, raw_offset, raw_length, agent_id, overflow_sha256) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16) `).run( event.event_id, event.session_id, event.timestamp, event.hook_event, event.cwd, event.repo_root ?? null, event.git_branch ?? null, metadata.turn_id ?? null, - metadata.tool_use_id ?? null, "", JSON.stringify(event), + metadata.tool_use_id ?? null, "", JSON.stringify(event), location?.segmentId ?? null, + location?.offset ?? null, location?.length ?? null, eventAgentId(event) ?? null, + overflowReferenceFromEvent(event)?.sha256 ?? null, ); if (insert.changes === 0) return { inserted: false, summaryTouched: false }; db.prepare(` @@ -298,6 +339,56 @@ export function indexEventInTransaction(db: DatabaseSync | undefined, event: Nor return { inserted: true, summaryTouched }; } +export function clearVerifiedRawJson(db: DatabaseSync, segmentId: string, batchSize = 500): number { + if (segmentId.length === 0 || !Number.isSafeInteger(batchSize) || batchSize <= 0) { + throw new TypeError("Raw JSON clearing requires a segment ID and positive batch size."); + } + const markerKey = `raw_json_clear_v1:${segmentId}`; + let cursor = Number(recordValue(db.prepare("SELECT value FROM index_metadata WHERE key = ?1").get(markerKey)).value ?? 0); + let cleared = 0; + for (;;) { + const rows = db.prepare(` + SELECT rowid, event_id, lcm_raw_json('', segment_id, raw_offset, raw_length) AS located_json + FROM events + WHERE segment_id = ?1 AND rowid > ?2 AND raw_json <> '' + ORDER BY rowid ASC LIMIT ?3 + `).all(segmentId, cursor, batchSize); + if (rows.length === 0) return cleared; + db.exec("BEGIN IMMEDIATE"); + try { + for (const row of rows) { + const record = recordValue(row); + const rowId = Number(record.rowid); + const eventId = String(record.event_id); + const event = decodePersistedEvent(String(record.located_json)); + if (event.event_id === eventId) { + cleared += Number(db.prepare("UPDATE events SET raw_json = '' WHERE rowid = ?1 AND event_id = ?2").run(rowId, eventId).changes); + } + cursor = rowId; + } + db.prepare(` + INSERT INTO index_metadata (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + `).run(markerKey, String(cursor)); + db.exec("COMMIT"); + } catch (error) { + const rollback = rollbackPreservingError(db, error); + if (rollback.kind === "rolled_back") throw error; + throw new AggregateError([error, rollback.rollbackError], "Raw JSON clearing rollback failed."); + } + } +} + +export function segmentsNeedRawJsonClearing(db: DatabaseSync, segmentIds: readonly string[]): boolean { + const marker = db.prepare("SELECT 1 FROM index_metadata WHERE key = ?1"); + return segmentIds.some((segmentId) => marker.get(`raw_json_clear_v1:${segmentId}`) === undefined); +} + +function eventAgentId(event: NormalizedEvent): string | undefined { + const value = event.payload.agent_id ?? event.payload.agentId; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + function indexFileRefsForEvent(db: DatabaseSync, event: NormalizedEvent): void { for (const ref of extractFileReferences(event)) { db.prepare(` @@ -320,19 +411,31 @@ function indexFileRefsForEvent(db: DatabaseSync, event: NormalizedEvent): void { function backfillExistingEventMetadata(db: DatabaseSync): void { const marker = recordValue(db.prepare("SELECT value FROM index_metadata WHERE key = ?1").get(EVENT_METADATA_BACKFILL_KEY)); if (marker.value === "1") return; - const rows = db.prepare("SELECT raw_json FROM events").all(); - const update = db.prepare("UPDATE events SET turn_id = ?1, tool_use_id = ?2 WHERE event_id = ?3"); + const rows = db.prepare(`SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events`).all(); + const update = db.prepare(` + UPDATE events SET turn_id = ?1, tool_use_id = ?2, agent_id = ?3, overflow_sha256 = ?4 WHERE event_id = ?5 + `); db.exec("BEGIN IMMEDIATE"); try { for (const row of rows) { const event = decodePersistedEvent(String(recordValue(row).raw_json)); const metadata = extractEventMetadata(event); - update.run(metadata.turn_id ?? null, metadata.tool_use_id ?? null, event.event_id); + update.run( + metadata.turn_id ?? null, + metadata.tool_use_id ?? null, + eventAgentId(event) ?? null, + overflowReferenceFromEvent(event)?.sha256 ?? null, + event.event_id, + ); } db.prepare(` INSERT INTO index_metadata (key, value) VALUES (?1, '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value `).run(EVENT_METADATA_BACKFILL_KEY); + db.prepare(` + INSERT INTO index_metadata (key, value) VALUES (?1, '1') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + `).run(EVENT_LOCATOR_METADATA_BACKFILL_KEY); db.exec("COMMIT"); } catch (error) { try { @@ -344,8 +447,30 @@ function backfillExistingEventMetadata(db: DatabaseSync): void { } } +export function backfillLocatorMetadata(db: DatabaseSync): void { + const marker = recordValue(db.prepare("SELECT value FROM index_metadata WHERE key = ?1").get(EVENT_LOCATOR_METADATA_BACKFILL_KEY)); + if (marker.value === "1") return; + const rows = db.prepare(`SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events ORDER BY rowid ASC`).all(); + const update = db.prepare("UPDATE events SET agent_id = ?1, overflow_sha256 = ?2 WHERE event_id = ?3"); + db.exec("BEGIN IMMEDIATE"); + try { + for (const row of rows) { + const event = decodePersistedEvent(String(recordValue(row).raw_json)); + update.run(eventAgentId(event) ?? null, overflowReferenceFromEvent(event)?.sha256 ?? null, event.event_id); + } + db.prepare(` + INSERT INTO index_metadata (key, value) VALUES (?1, '1') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + `).run(EVENT_LOCATOR_METADATA_BACKFILL_KEY); + db.exec("COMMIT"); + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } +} + function backfillExistingSessionMetadata(db: DatabaseSync): void { - const rows = db.prepare("SELECT raw_json FROM events WHERE hook_event = 'SessionStart'").all(); + const rows = db.prepare(`SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE hook_event = 'SessionStart'`).all(); const update = db.prepare("UPDATE sessions SET parent_session_id = ?2, agent_role = ?3, agent_nickname = ?4 WHERE session_id = ?1"); for (const row of rows) { const event = decodePersistedEvent(String(recordValue(row).raw_json)); @@ -359,7 +484,8 @@ export function backfillDelegationParents(db: DatabaseSync | undefined): string const marker = recordValue(db.prepare("SELECT value FROM index_metadata WHERE key = ?1").get(DELEGATION_PARENT_BACKFILL_KEY)); if (marker.value === "1") return undefined; const rows = db.prepare(` - SELECT e.raw_json FROM events e JOIN sessions s ON s.session_id = e.session_id + SELECT lcm_raw_json(e.raw_json, e.segment_id, e.raw_offset, e.raw_length) AS raw_json + FROM events e JOIN sessions s ON s.session_id = e.session_id WHERE e.hook_event = 'UserPromptSubmit' AND s.parent_session_id IS NULL ORDER BY e.timestamp ASC, e.rowid ASC `).all(); @@ -388,11 +514,8 @@ export function backfillFileRefs(db: DatabaseSync | undefined): string | undefin const marker = recordValue(db.prepare("SELECT value FROM index_metadata WHERE key = ?1").get(FILE_REF_BACKFILL_KEY)); if (marker.value === "1") return undefined; const rows = db.prepare(` - SELECT raw_json FROM events - WHERE hook_event = 'PostToolUse' AND ( - raw_json LIKE '%file_path%' OR raw_json LIKE '%filepath%' OR raw_json LIKE '%absolute_path%' - OR raw_json LIKE '%filename%' OR raw_json LIKE '%"path"%' OR raw_json LIKE '%"file"%' - ) ORDER BY timestamp ASC, rowid ASC + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events + WHERE hook_event = 'PostToolUse' ORDER BY timestamp ASC, rowid ASC `).all(); db.exec("BEGIN IMMEDIATE"); try { diff --git a/plugins/codex-lcm/src/storage-schema.ts b/plugins/codex-lcm/src/storage-schema.ts index f21ae34..d795d59 100644 --- a/plugins/codex-lcm/src/storage-schema.ts +++ b/plugins/codex-lcm/src/storage-schema.ts @@ -39,7 +39,12 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization turn_id TEXT, tool_use_id TEXT, text TEXT NOT NULL DEFAULT '', - raw_json TEXT NOT NULL + raw_json TEXT NOT NULL, + segment_id TEXT, + raw_offset INTEGER, + raw_length INTEGER, + agent_id TEXT, + overflow_sha256 TEXT ); CREATE VIRTUAL TABLE IF NOT EXISTS event_fts USING fts5( event_id UNINDEXED, @@ -123,6 +128,11 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization `); ensureColumn(db, "events", "turn_id", "TEXT"); ensureColumn(db, "events", "tool_use_id", "TEXT"); + ensureColumn(db, "events", "segment_id", "TEXT"); + ensureColumn(db, "events", "raw_offset", "INTEGER"); + ensureColumn(db, "events", "raw_length", "INTEGER"); + ensureColumn(db, "events", "agent_id", "TEXT"); + ensureColumn(db, "events", "overflow_sha256", "TEXT"); ensureColumn(db, "session_summaries", "summary_version", "INTEGER"); const backfillSessionMetadata = [ ensureColumn(db, "sessions", "parent_session_id", "TEXT"), @@ -143,6 +153,8 @@ export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization CREATE INDEX IF NOT EXISTS idx_events_tool_use ON events(session_id, tool_use_id, hook_event, timestamp); CREATE INDEX IF NOT EXISTS idx_events_session_hook_time ON events(session_id, hook_event, timestamp); CREATE INDEX IF NOT EXISTS idx_events_session_time ON events(session_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_events_agent_id ON events(agent_id); + CREATE INDEX IF NOT EXISTS idx_events_overflow_sha256 ON events(overflow_sha256); CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions(parent_session_id, last_seen); CREATE INDEX IF NOT EXISTS idx_sessions_last_seen ON sessions(last_seen); `); diff --git a/plugins/codex-lcm/src/storage-search.ts b/plugins/codex-lcm/src/storage-search.ts index 1c883ba..d26aef7 100644 --- a/plugins/codex-lcm/src/storage-search.ts +++ b/plugins/codex-lcm/src/storage-search.ts @@ -5,6 +5,7 @@ import type { NormalizedEvent } from "./events.ts"; import { overflowReferenceFromEvent, searchOverflowContent, type OverflowSearchMatch } from "./overflow.ts"; import { readRawEvents } from "./raw-log.ts"; import { parseStringArray, recordValue, rowToSessionSummary } from "./storage-rows.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import { getCurrentStoredSession, summarizeSessions } from "./storage-sessions.ts"; export { searchSummaryNodes } from "./storage-summaries.ts"; import type { SearchOverflowArgs, SearchSessionArgs, SessionDiscovery, SessionSearchMatch, SessionSummary } from "./storage-types.ts"; @@ -333,7 +334,8 @@ export function searchStoredSessions(db: DatabaseSync | undefined, rawLogPath: s let rows: unknown[] = []; const eventStatement = db.prepare(` SELECT s.*, - e.raw_json AS match_text, e.timestamp AS match_timestamp, 1 AS match_weight, + lcm_raw_json(e.raw_json, e.segment_id, e.raw_offset, e.raw_length) AS match_text, + e.timestamp AS match_timestamp, 1 AS match_weight, 'event' AS match_kind, e.event_id AS match_event_id FROM event_fts f JOIN events e ON e.event_id = f.event_id @@ -395,9 +397,9 @@ export function searchStoredOverflow( const limit = clampLimit(args.limit, 10, 50); const events = db ? db.prepare(` - SELECT raw_json + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events - WHERE json_extract(raw_json, '$.payload.overflow_ref.sha256') IS NOT NULL + WHERE overflow_sha256 IS NOT NULL AND (?1 IS NULL OR cwd = ?1) AND (?2 IS NULL OR repo_root = ?2) ORDER BY timestamp DESC, rowid DESC diff --git a/plugins/codex-lcm/src/storage-sessions.ts b/plugins/codex-lcm/src/storage-sessions.ts index f4642e0..51f93ba 100644 --- a/plugins/codex-lcm/src/storage-sessions.ts +++ b/plugins/codex-lcm/src/storage-sessions.ts @@ -4,6 +4,7 @@ import { decodePersistedEvent } from "./event-codec.ts"; import type { NormalizedEvent } from "./events.ts"; import { readRawEvents } from "./raw-log.ts"; import { isRecord, recordValue, rowToSessionSummary } from "./storage-rows.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import type { Health, LcmStats, @@ -439,8 +440,7 @@ export function resolveStoredSessionIdentifier( const row = recordValue(db.prepare(` SELECT session_id FROM events - WHERE json_extract(raw_json, '$.payload.agent_id') = ?1 - OR json_extract(raw_json, '$.payload.agentId') = ?1 + WHERE agent_id = ?1 ORDER BY timestamp DESC, rowid DESC LIMIT 1 `).get(trimmed)); @@ -466,12 +466,12 @@ export function getStoredSession( } const rows = limit === undefined ? db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 ORDER BY timestamp ASC, rowid ASC `).all(sessionId) : db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 ORDER BY timestamp ASC, rowid ASC LIMIT ?2 OFFSET ?3 diff --git a/plugins/codex-lcm/src/storage-summaries.ts b/plugins/codex-lcm/src/storage-summaries.ts index 40a638f..97b5d25 100644 --- a/plugins/codex-lcm/src/storage-summaries.ts +++ b/plugins/codex-lcm/src/storage-summaries.ts @@ -4,6 +4,7 @@ import { decodePersistedEvent } from "./event-codec.ts"; import type { NormalizedEvent } from "./events.ts"; import { readRawEvents } from "./raw-log.ts"; import { recordValue, rowToSessionMemorySummary, rowToSummaryNode } from "./storage-rows.ts"; +import { STORED_EVENT_JSON_SQL } from "./stored-event.ts"; import type { SearchSessionArgs } from "./storage-types.ts"; import { isCodexLcmToolEvent, isSummaryHook } from "./storage-sessions.ts"; import { @@ -212,7 +213,7 @@ export function getSummaryNodeSourceEvents( if (selectedIds.length === 0) return []; const placeholders = selectedIds.map((_, index) => `?${index + 1}`).join(", "); const rows = db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE event_id IN (${placeholders}) ORDER BY timestamp ASC, rowid ASC `).all(...selectedIds); @@ -236,7 +237,7 @@ export function getSessionSummarySourceEvents( if (!db || summary.source_event_ids.length === 0) return []; const placeholders = summary.source_event_ids.map((_, index) => `?${index + 1}`).join(", "); const events = db.prepare(` - SELECT raw_json + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE event_id IN (${placeholders}) ORDER BY timestamp ASC, rowid ASC @@ -441,7 +442,7 @@ function insertSummaryNode(db: DatabaseSync, node: SummaryNode): void { function getAllSummarySourceEventsForSession(db: DatabaseSync, sessionId: string): NormalizedEvent[] { return db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 AND hook_event IN ('UserPromptSubmit', 'Note', 'Stop', 'PreCompact', 'PostCompact') ORDER BY timestamp ASC, rowid ASC @@ -453,21 +454,21 @@ function getAllSummarySourceEventsForSession(db: DatabaseSync, sessionId: string function getSummaryEventsForSession(db: DatabaseSync, sessionId: string): NormalizedEvent[] { const earlySignals = db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 AND hook_event IN ${SUMMARY_SOURCE_HOOKS} ORDER BY timestamp ASC, rowid ASC LIMIT ?2 `).all(sessionId, SUMMARY_EARLY_SIGNAL_LIMIT); const latestSignals = db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 AND hook_event IN ${SUMMARY_SOURCE_HOOKS} ORDER BY timestamp DESC, rowid DESC LIMIT ?2 `).all(sessionId, SUMMARY_LATEST_SIGNAL_LIMIT); const recentEvents = db.prepare(` - SELECT raw_json FROM events + SELECT ${STORED_EVENT_JSON_SQL} AS raw_json FROM events WHERE session_id = ?1 ORDER BY timestamp DESC, rowid DESC LIMIT ?2 diff --git a/plugins/codex-lcm/src/storage-types.ts b/plugins/codex-lcm/src/storage-types.ts index 2326d34..345eaed 100644 --- a/plugins/codex-lcm/src/storage-types.ts +++ b/plugins/codex-lcm/src/storage-types.ts @@ -278,6 +278,13 @@ export type Health = { summary_count?: number; session_summary_count?: number; summary_node_count?: number; + storage_layout: "segmented-v1"; + migration_state: "none" | "pending" | "complete" | "error"; + active_bytes: number; + archive_bytes: number; + plain_segment_count: number; + compressed_segment_count: number; + config_error?: string; }; export type LcmStats = Health & { diff --git a/plugins/codex-lcm/src/storage.ts b/plugins/codex-lcm/src/storage.ts index 523a811..b9d1387 100644 --- a/plugins/codex-lcm/src/storage.ts +++ b/plugins/codex-lcm/src/storage.ts @@ -5,10 +5,13 @@ import { loadConfig, type LcmConfig } from "./config.ts"; import { createNoteEvent, type NormalizedEvent } from "./events.ts"; import type { FileReference } from "./file-refs.ts"; import type { OverflowReference, OverflowSearchMatch } from "./overflow.ts"; +import { cutOverLegacyLog, migrationInProgress } from "./maintenance.ts"; import { - appendRawEvents, - readRawEventIds, - readRawEvents, + appendSegmentedEvents, + readActiveRawEvents, + readAllLocatedRawEvents, + readAllRawEvents, + readAllRawLog, readRawLog, RawLogLockTimeoutError, withRawLogLock, @@ -43,6 +46,7 @@ import { currentRawLogState, emptyCleanupReport, indexedEventsById as readIndexedEventsById, + indexedActiveLogIsAppendOnly, indexedRawLogState as readIndexedRawLogState, indexEventInTransaction as indexStoredEventInTransaction, initializeIndex, @@ -78,6 +82,7 @@ import { storageStats, storedUsage, } from "./storage-sessions.ts"; +import { registerStoredEventReader } from "./stored-event.ts"; import { type SessionMemorySummary, type SummaryNode, @@ -109,16 +114,27 @@ export class LcmStorage { if (!this.readOnly) { fs.mkdirSync(this.config.home, { recursive: true, mode: 0o700 }); fs.chmodSync(this.config.home, 0o700); + cutOverLegacyLog(this.config); } if (this.readOnly && !fs.existsSync(this.config.indexPath)) { return; } try { this.db = new DatabaseSync(this.config.indexPath, { readOnly: this.readOnly, timeout: 5_000 }); + registerStoredEventReader(this.db, this.config); if (!this.readOnly) { fs.chmodSync(this.config.indexPath, 0o600); this.initialize(); - this.replayRawLogToIndex(); + if (migrationInProgress(this.config)) { + const indexedCount = Number(this.db.prepare("SELECT COUNT(*) AS count FROM events").get()?.count ?? 0); + if (indexedCount === 0) this.rebuildIndexFromRawStream(); + else this.replayActiveRawLogToIndex(); + } else { + if (!this.rawLogIsIndexed()) { + if (indexedActiveLogIsAppendOnly(this.db, this.config)) this.replayActiveRawLogToIndex(); + else this.rebuildIndexFromRawStream(); + } + } this.backfillDelegationParents(); this.backfillFileRefs(); this.backfillSessionMemorySummaries(); @@ -138,7 +154,7 @@ export class LcmStorage { if (this.db) { return this.db.prepare("SELECT 1 FROM events WHERE event_id = ?1 LIMIT 1").get(eventId) !== undefined; } - return readRawEvents(this.config.rawLogPath).some((event) => event.event_id === eventId); + return Array.from(readAllRawEvents(this.config)).some((event) => event.event_id === eventId); } ingest(event: NormalizedEvent): void { @@ -151,7 +167,7 @@ export class LcmStorage { if (!(error instanceof DerivedIndexError)) throw error; let rawDurable: boolean; try { - rawDurable = readRawEventIds(this.config.rawLogPath).has(event.event_id); + rawDurable = this.readRawEventIds().has(event.event_id); } catch { throw error; } @@ -199,11 +215,18 @@ export class LcmStorage { } if (eventsToAppend.length > 0) { - appendRawEvents(this.config.rawLogPath, eventsToAppend); + const locations = appendSegmentedEvents(this.config, eventsToAppend); this.storeRawEventIds(rawSeen); + return { + eventsToAppend, + locationsByEventId: new Map(eventsToAppend.map((event, index) => [event.event_id, locations[index]])), + rawLogState: rawLogWasIndexed ? this.rawLogState() : undefined, + skippedDuplicate, + }; } return { eventsToAppend, + locationsByEventId: new Map[number]>(), rawLogState: rawLogWasIndexed ? this.rawLogState() : undefined, skippedDuplicate, }; @@ -223,7 +246,11 @@ export class LcmStorage { for (const event of events) { if (indexSeen.has(event.event_id)) continue; indexSeen.add(event.event_id); - const result = this.indexEventInTransaction(event, { rebuildSummary: summaryRebuild === "event" }); + const result = this.indexEventInTransaction( + event, + { rebuildSummary: summaryRebuild === "event" }, + rawWrite.locationsByEventId.get(event.event_id), + ); if (result.summaryTouched) touchedSessions.add(event.session_id); } const rebuiltSessions = summaryRebuild === "sessions" @@ -249,9 +276,7 @@ export class LcmStorage { } private readRawEventIds(): Set { - const result = readCachedRawEventIds(this.config.rawLogPath, this.rawEventIdCache); - this.rawEventIdCache = result.cache; - return result.eventIds; + return new Set(Array.from(readAllRawEvents(this.config), (event) => event.event_id)); } private storeRawEventIds(eventIds: Set): void { @@ -320,6 +345,7 @@ export class LcmStorage { private reopenWritableIndex(): void { this.db?.close(); this.db = new DatabaseSync(this.config.indexPath, { timeout: 5_000 }); + registerStoredEventReader(this.db, this.config); } health(): Health { @@ -353,7 +379,7 @@ export class LcmStorage { if (!this.db) return; if (this.rawLogIsIndexed()) return; const snapshot = withRawLogLock(this.config.rawLogPath, () => ({ - rawLog: readRawLog(this.config.rawLogPath), + rawLog: readAllRawLog(this.config), state: this.rawLogState(), })); const rawLog = snapshot.rawLog; @@ -365,7 +391,7 @@ export class LcmStorage { this.indexError = `Raw JSONL contains ${rawLog.malformedLineCount} malformed ${noun}; destructive index reconciliation is disabled until the log is repaired.`; } if (rawEvents.length === 0) { - if (indexedIds.size > 0 && rawLog.malformedLineCount === 0) this.rebuildIndexFromRawEvents([], snapshot.state); + if (indexedIds.size > 0 && rawLog.malformedLineCount === 0) this.rebuildIndexFromRawStream(); else if (rawLog.malformedLineCount === 0) this.recordRawLogState(snapshot.state); return; } @@ -376,7 +402,7 @@ export class LcmStorage { return indexedRaw !== undefined && indexedRaw !== JSON.stringify(event); }); if ((hasStaleIndexedRows || hasChangedIndexedRows) && rawLog.malformedLineCount === 0) { - this.rebuildIndexFromRawEvents(rawEvents, snapshot.state); + this.rebuildIndexFromRawStream(); return; } const missingEvents = rawEvents.filter((event) => !indexedIds.has(event.event_id)); @@ -386,11 +412,13 @@ export class LcmStorage { } const touchedSessions = new Set(); + const missingIds = new Set(missingEvents.map((event) => event.event_id)); this.db.exec("BEGIN IMMEDIATE"); try { - for (const event of missingEvents) { - const result = this.indexEventInTransaction(event, { rebuildSummary: false }); - if (result.summaryTouched) touchedSessions.add(event.session_id); + for (const located of readAllLocatedRawEvents(this.config)) { + if (!missingIds.has(located.event.event_id)) continue; + const result = this.indexEventInTransaction(located.event, { rebuildSummary: false }, located.location); + if (result.summaryTouched) touchedSessions.add(located.event.session_id); } this.rebuildTouchedSummarySessions(touchedSessions); if (rawLog.malformedLineCount === 0) this.recordRawLogState(snapshot.state); @@ -401,15 +429,41 @@ export class LcmStorage { } } - private rebuildIndexFromRawEvents(rawEvents: NormalizedEvent[], state: RawLogState): void { + private replayActiveRawLogToIndex(): void { if (!this.db) return; + const rawLog = withRawLogLock(this.config.rawLogPath, () => readRawLog(this.config.rawLogPath)); + if (rawLog.malformedLineCount > 0) { + this.indexError = `Active raw JSONL contains ${rawLog.malformedLineCount} malformed lines.`; + } + const indexedIds = this.knownEventIds(rawLog.events.map((event) => event.event_id)); + const missingIds = new Set(rawLog.events.filter((event) => !indexedIds.has(event.event_id)).map((event) => event.event_id)); + if (missingIds.size === 0) return; + const touchedSessions = new Set(); + this.db.exec("BEGIN IMMEDIATE"); + try { + for (const located of readActiveRawEvents(this.config)) { + if (!missingIds.has(located.event.event_id)) continue; + const result = this.indexEventInTransaction(located.event, { rebuildSummary: false }, located.location); + if (result.summaryTouched) touchedSessions.add(located.event.session_id); + } + this.rebuildTouchedSummarySessions(touchedSessions); + this.db.exec("COMMIT"); + } catch (error) { + const failure = rollbackPreservingError(this.db, error).original; + this.indexError = failure instanceof Error ? failure.message : String(failure); + } + } + + private rebuildIndexFromRawStream(): void { + if (!this.db) return; + const state = withRawLogLock(this.config.rawLogPath, () => this.rawLogState()); const touchedSessions = new Set(); this.db.exec("BEGIN IMMEDIATE"); try { this.clearDerivedIndex(); - for (const event of rawEvents) { - const result = this.indexEventInTransaction(event, { rebuildSummary: false }); - if (result.summaryTouched) touchedSessions.add(event.session_id); + for (const located of readAllLocatedRawEvents(this.config)) { + const result = this.indexEventInTransaction(located.event, { rebuildSummary: false }, located.location); + if (result.summaryTouched) touchedSessions.add(located.event.session_id); } this.rebuildTouchedSummarySessions(touchedSessions); this.recordRawLogState(state); @@ -417,7 +471,9 @@ export class LcmStorage { } catch (error) { const rollback = rollbackPreservingError(this.db, error); const message = error instanceof Error ? error.message : String(error); - this.indexError = rollback.kind === "rolled_back" ? message : `${message}; rollback failed: ${rollback.rollbackError instanceof Error ? rollback.rollbackError.message : String(rollback.rollbackError)}`; + this.indexError = rollback.kind === "rolled_back" + ? message + : `${message}; rollback failed: ${rollback.rollbackError instanceof Error ? rollback.rollbackError.message : String(rollback.rollbackError)}`; } } @@ -434,7 +490,7 @@ export class LcmStorage { } private rawLogIsIndexed(): boolean { - return this.db ? isRawLogIndexed(this.db, this.config.rawLogPath) : false; + return this.db ? isRawLogIndexed(this.db, this.config) : false; } private indexedRawLogState(): string | undefined { @@ -446,7 +502,7 @@ export class LcmStorage { } private rawLogState(): RawLogState { - return currentRawLogState(this.config.rawLogPath); + return currentRawLogState(this.config); } private rebuildTouchedSummarySessions(sessionIds: Iterable): string[] { @@ -596,8 +652,12 @@ export class LcmStorage { if (this.db) initializeIndex(this.db); } - private indexEventInTransaction(event: NormalizedEvent, options: { rebuildSummary: boolean }): IndexEventResult { - return indexStoredEventInTransaction(this.db, event, options.rebuildSummary); + private indexEventInTransaction( + event: NormalizedEvent, + options: { rebuildSummary: boolean }, + location?: ReturnType[number], + ): IndexEventResult { + return indexStoredEventInTransaction(this.db, event, options.rebuildSummary, location); } private backfillDelegationParents(): void { diff --git a/plugins/codex-lcm/src/stored-event.ts b/plugins/codex-lcm/src/stored-event.ts new file mode 100644 index 0000000..924482f --- /dev/null +++ b/plugins/codex-lcm/src/stored-event.ts @@ -0,0 +1,17 @@ +import type { DatabaseSync } from "node:sqlite"; + +import type { LcmConfig } from "./config.ts"; +import { createLocatedEventReader } from "./raw-log.ts"; + +export const STORED_EVENT_JSON_SQL = "lcm_raw_json(raw_json, segment_id, raw_offset, raw_length)" as const; + +export function registerStoredEventReader(db: DatabaseSync, config: LcmConfig): void { + const readLocatedEvent = createLocatedEventReader(config); + db.function("lcm_raw_json", { directOnly: true }, (rawJson, segmentId, offset, length) => { + if (typeof rawJson === "string" && rawJson.length > 0) return rawJson; + if (typeof segmentId !== "string" || typeof offset !== "number" || typeof length !== "number") { + throw new TypeError("Stored event has neither inline JSON nor a valid raw locator."); + } + return JSON.stringify(readLocatedEvent({ segmentId, offset, length })); + }); +} diff --git a/plugins/codex-lcm/tests/storage.test.ts b/plugins/codex-lcm/tests/storage.test.ts index 138223a..f7cfe77 100644 --- a/plugins/codex-lcm/tests/storage.test.ts +++ b/plugins/codex-lcm/tests/storage.test.ts @@ -6,14 +6,457 @@ import { DatabaseSync } from "node:sqlite"; import test from "node:test"; import { Worker } from "node:worker_threads"; +import { loadConfig } from "../src/config.ts"; import { normalizeHookEvent, type NormalizedEvent } from "../src/events.ts"; -import { appendRawEvents, readRawLog, withRawLogLock } from "../src/raw-log.ts"; +import { runMaintenanceOnce } from "../src/maintenance.ts"; +import { + appendRawEvents, + appendSegmentedEvents, + readAllRawEvents, + readLocatedEvent, + readRawLog, + withRawLogLock, +} from "../src/raw-log.ts"; +import { readManifest, segmentStoreState, writeManifestAtomic, type SegmentManifest } from "../src/raw-segments.ts"; import { sha256 } from "../src/redact.ts"; +import { recordValue } from "../src/storage-rows.ts"; +import { clearVerifiedRawJson, segmentsNeedRawJsonClearing } from "../src/storage-persistence.ts"; +import { registerStoredEventReader } from "../src/stored-event.ts"; import { createStorage, LcmStorage } from "../src/storage.ts"; import { clearDerivedSummaries, readJsonl, tempHome } from "./helpers.ts"; const now = () => new Date("2026-06-09T12:00:00.000Z"); +test("retention configuration reads valid .env values and rejects invalid values", () => { + const missingHome = tempHome(); + assert.equal(loadConfig({ home: missingHome, env: {} }).retentionDays, undefined); + + const validHome = tempHome(); + fs.writeFileSync(path.join(validHome, ".env"), "# local retention\nCODEX_LCM_RETENTION_DAYS=90\n"); + const config = loadConfig({ home: validHome, env: {} }); + assert.equal(config.retentionDays, 90); + assert.equal(config.configError, undefined); + assert.equal(loadConfig({ home: validHome, env: { CODEX_LCM_RETENTION_DAYS: "30" } }).retentionDays, 30); + + for (const value of ["0", "-1", "2.5", "ninety"]) { + const home = tempHome(); + fs.writeFileSync(path.join(home, ".env"), `CODEX_LCM_RETENTION_DAYS=${value}\n`); + const invalid = loadConfig({ home, env: {} }); + assert.equal(invalid.retentionDays, undefined, value); + assert.equal(invalid.configError, "CODEX_LCM_RETENTION_DAYS must be a positive integer.", value); + } + const unsafe = loadConfig({ home: tempHome(), env: { CODEX_LCM_RETENTION_DAYS: "9".repeat(400) } }); + assert.equal(unsafe.retentionDays, undefined); + assert.equal(unsafe.configError, "CODEX_LCM_RETENTION_DAYS must be a positive safe integer."); + + const duplicateHome = tempHome(); + fs.writeFileSync(duplicateHome + "/.env", "CODEX_LCM_RETENTION_DAYS=90\nCODEX_LCM_RETENTION_DAYS=91\n"); + const duplicate = loadConfig({ home: duplicateHome, env: {} }); + assert.equal(duplicate.retentionDays, undefined); + assert.notEqual(duplicate.configError, undefined); +}); + +test("segment manifest defaults, validates, writes atomically, and changes store state", () => { + const home = tempHome(); + const manifestPath = path.join(home, "segments", "manifest.json"); + assert.deepEqual(readManifest(manifestPath), { version: 1, segments: [] }); + + const manifest: SegmentManifest = { + version: 1, + segments: [{ + id: "2026-06-09-000001", + path: "segments/2026-06-09-000001.jsonl", + compressed: false, + byte_count: 42, + event_count: 1, + first_timestamp: "2026-06-09T12:00:00.000Z", + last_timestamp: "2026-06-09T12:00:00.000Z", + sha256: "a".repeat(64), + }], + }; + writeManifestAtomic(manifestPath, manifest); + assert.deepEqual(readManifest(manifestPath), manifest); + assert.notEqual( + segmentStoreState(manifest), + segmentStoreState({ ...manifest, segments: [{ ...manifest.segments[0], sha256: "b".repeat(64) }] }), + ); + + fs.writeFileSync(manifestPath, "{not json"); + assert.throws(() => readManifest(manifestPath), /manifest/u); + assert.equal(fs.readFileSync(manifestPath, "utf8"), "{not json"); + fs.writeFileSync(manifestPath, JSON.stringify({ ...manifest, segments: [{ ...manifest.segments[0], path: "../escape.jsonl" }] })); + assert.throws(() => readManifest(manifestPath), /manifest/u); +}); + +test("rotates active raw log before a record exceeds the configured cap", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const seed = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "segment-seed", cwd: "/tmp/segment-seed", prompt: "seed" }), + env: {}, + now, + }); + const next = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "segment-next", cwd: "/tmp/segment-next", prompt: "next" }), + env: {}, + now, + }); + appendRawEvents(config.rawLogPath, [seed]); + const cap = fs.statSync(config.rawLogPath).size + 1; + + appendSegmentedEvents(config, [next], { segmentCapBytes: cap }); + + const manifest = readManifest(config.manifestPath); + assert.equal(manifest.segments.length, 1); + assert.equal(manifest.segments[0].event_count, 1); + assert.equal(manifest.segments[0].sha256, sha256(fs.readFileSync(path.join(home, manifest.segments[0].path)))); + assert.deepEqual(readJsonl(path.join(home, manifest.segments[0].path)), [seed]); + assert.deepEqual(readJsonl(config.rawLogPath), [next]); +}); + +test("reads located raw event from a closed segment", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const seed = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "located-seed", cwd: "/tmp/located-seed", prompt: "seed" }), + env: {}, + now, + }); + const next = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "located-next", cwd: "/tmp/located-next", prompt: "next" }), + env: {}, + now, + }); + appendRawEvents(config.rawLogPath, [seed]); + appendSegmentedEvents(config, [next], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + const record = readManifest(config.manifestPath).segments[0]; + const length = fs.statSync(path.join(home, record.path)).size; + + assert.deepEqual(readLocatedEvent(config, { segmentId: record.id, offset: 0, length }), seed); +}); + +test("keeps active event locators valid after rotation", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const first = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "stable-first", cwd: "/tmp/stable-first", prompt: "first" }), + env: {}, + now, + }); + const second = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "stable-second", cwd: "/tmp/stable-second", prompt: "second" }), + env: {}, + now, + }); + const firstLocation = appendSegmentedEvents(config, [first], { segmentCapBytes: 1_024 })[0]; + + appendSegmentedEvents(config, [second], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + + assert.deepEqual(readLocatedEvent(config, firstLocation), first); +}); + +test("streams segmented history across closed and active logs", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const seed = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "stream-seed", cwd: "/tmp/stream-seed", prompt: "seed" }), + env: {}, + now, + }); + const next = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "stream-next", cwd: "/tmp/stream-next", prompt: "next" }), + env: {}, + now, + }); + appendRawEvents(config.rawLogPath, [seed]); + appendSegmentedEvents(config, [next], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + const eventIds: string[] = []; + for (const event of readAllRawEvents(config)) eventIds.push(event.event_id); + assert.deepEqual(eventIds, [seed.event_id, next.event_id]); +}); + +test("hydrates an event from its raw locator after SQLite payload clearing", () => { + const home = tempHome(); + const event = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "cold-locator", cwd: "/tmp/cold-locator", prompt: "keep me" }), + env: {}, + now, + }); + const storage = createStorage({ home }); + storage.ingest(event); + storage.close(); + const next = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "cold-locator-next", cwd: "/tmp/cold-locator", prompt: "rotate" }), + env: {}, + now, + }); + const config = loadConfig({ home }); + appendSegmentedEvents(config, [next], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + + const db = new DatabaseSync(path.join(home, "index.sqlite")); + registerStoredEventReader(db, config); + const row = recordValue(db.prepare(` + SELECT segment_id, raw_offset, raw_length, raw_json FROM events WHERE event_id = ?1 + `).get(event.event_id)); + assert.equal(row.segment_id, "0000000000000001"); + assert.equal(row.raw_offset, 0); + assert.equal(typeof row.raw_length, "number"); + assert.equal(typeof row.raw_json, "string"); + assert.equal(clearVerifiedRawJson(db, String(row.segment_id), 1), 1); + assert.equal(db.prepare("SELECT raw_json FROM events WHERE event_id = ?1").get(event.event_id)?.raw_json, ""); + db.close(); + + const reopened = createStorage({ home, readOnly: true }); + try { + assert.deepEqual(reopened.getSession(event.session_id).events, [event]); + assert.deepEqual(reopened.getRecentContext({ sessionId: event.session_id }).events, [event]); + assert.equal(reopened.getSessionGraph(event.session_id).nodes.some((node) => node.event_id === event.event_id), true); + assert.equal(reopened.searchSessions({ query: "keep me" }).some((match) => match.session_id === event.session_id), true); + assert.equal(reopened.packContext({ sessionIds: [event.session_id], query: "keep me" }).sources.some( + (source) => source.kind === "event" && source.event_id === event.event_id, + ), true); + } finally { + reopened.close(); + } +}); + +test("automatically cuts over a legacy raw log without dropping indexed history", () => { + const home = tempHome(); + const event = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "legacy-cutover", cwd: "/tmp/legacy-cutover", prompt: "old history" }), + env: {}, + now, + }); + const legacy = createStorage({ home }); + legacy.ingest(event); + legacy.close(); + const config = loadConfig({ home }); + fs.unlinkSync(config.manifestPath); + + const reopened = createStorage({ home }); + try { + const manifest = readManifest(config.manifestPath); + assert.deepEqual(manifest.migration, { + legacy_path: "segments/legacy.jsonl", + offset: 0, + complete: false, + }); + assert.equal(fs.statSync(config.rawLogPath).size, 0); + assert.equal(fs.existsSync(path.join(home, "segments", "legacy.jsonl")), true); + assert.deepEqual(reopened.getSession(event.session_id).events, [event]); + } finally { + reopened.close(); + } +}); + +test("recovers a legacy cutover interrupted after the authoritative log rename", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const event = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "legacy-cutover-recovery", cwd: "/tmp/legacy-cutover", prompt: "recover me" }), + env: {}, + now, + }); + appendRawEvents(config.rawLogPath, [event]); + fs.mkdirSync(config.segmentsDir, { recursive: true }); + fs.renameSync(config.rawLogPath, path.join(config.segmentsDir, "legacy.jsonl")); + + const recovered = createStorage({ config }); + try { + assert.equal(readManifest(config.manifestPath).migration?.complete, false); + assert.equal(fs.existsSync(config.rawLogPath), true); + assert.deepEqual(recovered.getSession(event.session_id).events, [event]); + } finally { + recovered.close(); + } +}); + +test("resumes legacy migration without duplicating events", () => { + const home = tempHome(); + const legacy = createStorage({ home }); + const events = ["one", "two", "three"].map((prompt, index) => normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "legacy-resume", cwd: "/tmp/legacy-resume", prompt }), + env: {}, + now: () => new Date(`2026-06-09T12:00:0${index}.000Z`), + })); + legacy.ingestMany(events); + legacy.close(); + const config = loadConfig({ home }); + fs.unlinkSync(config.manifestPath); + const cutover = createStorage({ home }); + cutover.close(); + + const first = runMaintenanceOnce(config, { maxSegments: 1, segmentCapBytes: Buffer.byteLength(JSON.stringify(events[0])) + 1 }); + assert.equal(first.migrated, 1); + assert.equal(readManifest(config.manifestPath).migration?.complete, false); + + const second = runMaintenanceOnce(config, { segmentCapBytes: Buffer.byteLength(JSON.stringify(events[0])) + 1 }); + assert.equal(second.migrated, 2); + assert.equal(readManifest(config.manifestPath).migration?.complete, true); + assert.deepEqual(Array.from(readAllRawEvents(config)).map((event) => event.event_id), events.map((event) => event.event_id)); +}); + +test("keeps earlier quarantine findings when a resumed migration reaches EOF", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const events = ["before", "after"].map((prompt, index) => normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "legacy-quarantine", cwd: "/tmp/legacy-quarantine", prompt }), + env: {}, + now: () => new Date(`2026-06-09T12:00:0${index}.000Z`), + })); + fs.writeFileSync(config.rawLogPath, `${JSON.stringify(events[0])}\nnot-json\n${JSON.stringify(events[1])}\n`); + const cutover = createStorage({ config }); + cutover.close(); + const segmentCapBytes = Buffer.byteLength(JSON.stringify(events[0])) + Buffer.byteLength("\nnot-json\n"); + + const first = runMaintenanceOnce(config, { maxSegments: 1, segmentCapBytes }); + assert.equal(first.quarantined, 1); + assert.equal(readManifest(config.manifestPath).migration?.complete, false); + + const second = runMaintenanceOnce(config, { segmentCapBytes }); + assert.equal(second.quarantined, 1); + assert.deepEqual(second.errors, ["1 malformed legacy records were quarantined."]); + assert.equal(readManifest(config.manifestPath).migration?.complete, true); + assert.equal(fs.existsSync(path.join(config.segmentsDir, "legacy.jsonl")), true); +}); + +test("compresses a verified closed segment and keeps locator reads available", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const first = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "compressed-locator", cwd: "/tmp/compressed-locator", prompt: "archive me" }), + env: {}, + now, + }); + const firstLocation = appendSegmentedEvents(config, [first], { segmentCapBytes: 1_024 })[0]; + const next = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "compressed-next", cwd: "/tmp/compressed-locator", prompt: "rotate" }), + env: {}, + now, + }); + appendSegmentedEvents(config, [next], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + + const report = runMaintenanceOnce(config); + + assert.equal(report.compressed, 1); + const record = readManifest(config.manifestPath).segments[0]; + assert.equal(record.compressed, true); + assert.equal(fs.existsSync(path.join(home, record.path)), true); + assert.deepEqual(readLocatedEvent(config, firstLocation), first); +}); + +test("rebuilds a deleted index with locators that allow archived payload clearing", () => { + const home = tempHome(); + const config = loadConfig({ home }); + const archived = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "locator-rebuild", cwd: "/tmp/locator-rebuild", prompt: "archive" }), + env: {}, + now, + }); + const storage = createStorage({ config }); + storage.ingest(archived); + storage.close(); + const active = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "locator-rebuild-active", cwd: "/tmp/locator-rebuild", prompt: "active" }), + env: {}, + now, + }); + appendSegmentedEvents(config, [active], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + runMaintenanceOnce(config); + fs.unlinkSync(config.indexPath); + + const rebuilt = createStorage({ config }); + rebuilt.close(); + const beforeMaintenance = new DatabaseSync(config.indexPath); + assert.equal(segmentsNeedRawJsonClearing(beforeMaintenance, readManifest(config.manifestPath).segments.map((record) => record.id)), true); + beforeMaintenance.close(); + runMaintenanceOnce(config); + + const db = new DatabaseSync(config.indexPath, { readOnly: true }); + const row = db.prepare("SELECT raw_json, segment_id, raw_offset, raw_length FROM events WHERE event_id = ?1").get(archived.event_id); + assert.equal(row?.raw_json, ""); + assert.equal(typeof row?.segment_id, "string"); + assert.equal(typeof row?.raw_offset, "number"); + assert.equal(typeof row?.raw_length, "number"); + assert.equal(segmentsNeedRawJsonClearing(db, readManifest(config.manifestPath).segments.map((record) => record.id)), false); + db.close(); + const reopened = createStorage({ config, readOnly: true }); + try { + assert.deepEqual(reopened.getSession(archived.session_id).events, [archived]); + } finally { + reopened.close(); + } +}); + +test("expires configured raw history while preserving session summaries", () => { + const home = tempHome(); + fs.writeFileSync(path.join(home, ".env"), "CODEX_LCM_RETENTION_DAYS=30\n", { mode: 0o600 }); + const config = loadConfig({ home }); + const oldEvent = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "retained-summary", cwd: "/tmp/retention", prompt: "old detail" }), + env: {}, + now: () => new Date("2026-06-01T00:00:00.000Z"), + }); + const storage = createStorage({ config }); + storage.ingest(oldEvent); + storage.close(); + const next = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "retention-active", cwd: "/tmp/retention", prompt: "active" }), + env: {}, + now: () => new Date("2026-08-06T00:00:00.000Z"), + }); + appendSegmentedEvents(config, [next], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + + const report = runMaintenanceOnce(config, { now: () => new Date("2026-08-06T00:00:00.000Z") }); + + assert.equal(report.expired, 1); + assert.equal(readManifest(config.manifestPath).segments.length, 0); + const db = new DatabaseSync(config.indexPath, { readOnly: true }); + assert.equal(db.prepare("SELECT COUNT(*) AS count FROM events WHERE event_id = ?1").get(oldEvent.event_id)?.count, 0); + assert.equal(db.prepare("SELECT COUNT(*) AS count FROM sessions WHERE session_id = ?1").get(oldEvent.session_id)?.count, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS count FROM session_summaries WHERE session_id = ?1").get(oldEvent.session_id)?.count, 1); + db.close(); +}); + +test("invalid retention configuration blocks source deletion", () => { + const home = tempHome(); + fs.writeFileSync(path.join(home, ".env"), "CODEX_LCM_RETENTION_DAYS=0\n", { mode: 0o600 }); + const config = loadConfig({ home }); + const event = normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ session_id: "invalid-retention", cwd: "/tmp/retention", prompt: "preserve" }), + env: {}, + now: () => new Date("2026-06-01T00:00:00.000Z"), + }); + appendSegmentedEvents(config, [event], { segmentCapBytes: 1_024 }); + appendSegmentedEvents(config, [event], { segmentCapBytes: fs.statSync(config.rawLogPath).size + 1 }); + + const report = runMaintenanceOnce(config, { now: () => new Date("2026-08-06T00:00:00.000Z") }); + + assert.deepEqual(report.errors, ["CODEX_LCM_RETENTION_DAYS must be a positive integer."]); + assert.equal(readManifest(config.manifestPath).segments.length, 1); +}); + test("writable storage restricts its home and SQLite index permissions", () => { if (process.platform === "win32") return; const home = tempHome(); @@ -975,7 +1418,7 @@ test("constructor replay leaves an interleaved raw append visible to the next op const reopened = createStorage({ home }); // Then: the append that raced replay is present in both authority and index. - assert.equal(readRawLog(rawLogPath).events.length, 2); + assert.equal(Array.from(readAllRawEvents(loadConfig({ home }))).length, 2); assert.equal(reopened.health().event_count, 2); assert.equal(reopened.hasEvent(interleaved.event_id), true); reopened.close(); @@ -1759,7 +2202,7 @@ test("cleanup acquires the write lock before snapshotting searchable events", () storage.close(); const beginIndex = calls.findIndex((sql) => sql === "BEGIN IMMEDIATE"); - const snapshotIndex = calls.findIndex((sql) => sql.includes("SELECT raw_json") && sql.includes("FROM events")); + const snapshotIndex = calls.findIndex((sql) => sql.includes("AS raw_json") && sql.includes("FROM events")); const optimizeIndex = calls.findIndex((sql) => sql === "INSERT INTO event_fts(event_fts) VALUES('optimize')"); const vacuumIndex = calls.findIndex((sql) => sql === "VACUUM"); assert.equal(beginIndex >= 0, true);