diff --git a/scripts/lib/codex-app-server-watchdog-proxy.mjs b/scripts/lib/codex-app-server-watchdog-proxy.mjs index fa481bb7..4be7c763 100644 --- a/scripts/lib/codex-app-server-watchdog-proxy.mjs +++ b/scripts/lib/codex-app-server-watchdog-proxy.mjs @@ -1,4 +1,4 @@ -import { createProgressWatchdog } from "./agent-progress-watchdog.mjs"; +import { createProgressWatchdog } from "./watchdog-investigation-progress.mjs"; import { observeCodexAppServerMessage } from "./codex-progress-watchdog.mjs"; const STREAM_WATCHDOG_DEFAULTS = Object.freeze({ @@ -119,4 +119,4 @@ export function createAppServerWatchdogRouter(options = {}) { onServerMessage, activeTurnCount: () => turns.size, }; -} \ No newline at end of file +} diff --git a/scripts/lib/codex-progress-watchdog.mjs b/scripts/lib/codex-progress-watchdog.mjs index 2f18b691..258f1bc7 100644 --- a/scripts/lib/codex-progress-watchdog.mjs +++ b/scripts/lib/codex-progress-watchdog.mjs @@ -1,4 +1,4 @@ -import { createProgressWatchdog } from "./agent-progress-watchdog.mjs"; +import { createProgressWatchdog } from "./watchdog-investigation-progress.mjs"; import { classifyAppServerItem, isSuccessfulAppServerItem, @@ -94,6 +94,43 @@ function resetMicroNarration(context) { context.microNarrationIntentCount = 0; } +function appServerEvidence(item = {}) { + if (item?.type === "commandExecution") { + return { + toolName: "commandExecution", + input: { + command: item.command ?? item.commandText ?? item.process?.command ?? "", + }, + response: + item.aggregatedOutput ?? + item.output ?? + item.stdout ?? + item.result ?? + item.text ?? + item.content ?? + "", + }; + } + return { + toolName: + item?.appContext?.actionName || + item?.toolName || + item?.tool || + item?.name || + item?.server || + "", + input: item?.arguments ?? item?.input ?? {}, + response: + item?.aggregatedOutput ?? + item?.output ?? + item?.stdout ?? + item?.result ?? + item?.text ?? + item?.content ?? + "", + }; +} + function observeMicroNarration(delta, context) { if (typeof delta !== "string" || delta.length === 0) return { action: "allow" }; const current = `${context.microNarrationBuffer || ""}${delta}`; @@ -189,6 +226,14 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) { } const classification = classifyAppServerItem(item); if (classification.kind === "evidence") { + if (typeof watchdog.prepareEvidenceAttempt === "function") { + const evidence = appServerEvidence(item); + watchdog.prepareEvidenceAttempt({ + toolName: evidence.toolName, + input: evidence.input, + volatility: classification.volatility || "stable", + }); + } const decision = watchdog.chargeEvidenceAttempt(); if (decision.action === "block") { return maybeInterrupt( @@ -212,6 +257,17 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) { } else if (classification.kind === "execution") { watchdog.recordExecutionProgress({ kind: "codex_execution_completed" }); resetMicroNarration(context); + } else if ( + classification.kind === "evidence" && + typeof watchdog.recordEvidenceResult === "function" + ) { + const evidence = appServerEvidence(item); + watchdog.recordEvidenceResult({ + toolName: evidence.toolName, + input: evidence.input, + volatility: classification.volatility || "stable", + response: evidence.response, + }); } } return { decision: { action: "allow" } }; diff --git a/scripts/lib/codex-watchdog-hook.mjs b/scripts/lib/codex-watchdog-hook.mjs index 7ece5c0b..752da86a 100644 --- a/scripts/lib/codex-watchdog-hook.mjs +++ b/scripts/lib/codex-watchdog-hook.mjs @@ -1,4 +1,4 @@ -import { createProgressWatchdog } from "./agent-progress-watchdog.mjs"; +import { createProgressWatchdog } from "./watchdog-investigation-progress.mjs"; import { createEvidenceRegistry, deriveShellEvidenceDescriptor, @@ -34,6 +34,7 @@ function hydrate(state, options) { volatileReadIntervalMs: options.volatileReadIntervalMs, evidenceSoftLimit: options.evidenceSoftLimit, evidenceHardLimit: options.evidenceHardLimit, + investigationCreditLimit: options.investigationCreditLimit, }; if (options.generatedCharSoftLimit !== undefined) { watchdogOptions.generatedCharSoftLimit = options.generatedCharSoftLimit; @@ -198,6 +199,7 @@ export function evaluateCodexHook(input, state = {}, options = {}) { maxSubagentInputChars: options.maxSubagentInputChars ?? 6_000, evidenceSoftLimit: options.evidenceSoftLimit ?? 8, evidenceHardLimit: options.evidenceHardLimit ?? 12, + investigationCreditLimit: options.investigationCreditLimit, maxNarrationRecoveryAttempts: options.maxNarrationRecoveryAttempts ?? DEFAULT_MAX_NARRATION_RECOVERY_ATTEMPTS, generatedCharSoftLimit: stopFinalizationCandidate @@ -285,6 +287,12 @@ export function evaluateCodexHook(input, state = {}, options = {}) { } else if (classification.kind === "execution") { watchdog.recordExecutionProgress({ kind: "tool_execution_completed", toolName: input.tool_name }); } else if (classification.kind === "evidence" && !responseExplicitlyFailed(input.tool_response)) { + watchdog.recordEvidenceResult({ + toolName: input.tool_name, + input: input.tool_input, + volatility: classification.volatility || "stable", + response: input.tool_response, + }); const descriptor = shellEvidenceDescriptor(input); if (descriptor) { evidenceRegistry.record({ diff --git a/scripts/lib/watchdog-investigation-progress.mjs b/scripts/lib/watchdog-investigation-progress.mjs new file mode 100644 index 00000000..bfd19998 --- /dev/null +++ b/scripts/lib/watchdog-investigation-progress.mjs @@ -0,0 +1,187 @@ +import { posix } from "node:path"; + +import { createProgressWatchdog as createBaseProgressWatchdog } from "./agent-progress-watchdog.mjs"; + +const DEFAULT_INVESTIGATION_CREDIT_LIMIT = 4; +const SOURCE_READ_COMMAND = /\b(?:get-content|cat|type)\b\s+(?:(?:-path|-literalpath)\s+)?(?:"([^"]+)"|'([^']+)'|([^\s|;&]+))/i; +const SOURCE_PATH = /\.(?:[cm]?[jt]sx?|json|ya?ml|toml|md|ps1|cs|csproj|fs|go|rs|py|rb|php|java|kt|kts|swift|c|cc|cpp|cxx|h|hh|hpp|sh|bash|zsh)$/i; + +function asText(value) { + if (typeof value === "string") return value; + try { + return JSON.stringify(value ?? null); + } catch { + return String(value ?? ""); + } +} + +function normalizeSourcePath(value) { + let path = String(value || "").trim().replace(/^['"]|['"]$/g, "").replace(/\\/g, "/"); + if (!path || /^https?:\/\//i.test(path)) return null; + path = posix.normalize(path).replace(/^\.\//, ""); + return SOURCE_PATH.test(path) ? path : null; +} + +function sourceReadTarget(toolName, input = {}) { + const direct = input?.path ?? input?.file_path ?? input?.filePath ?? input?.filename; + const directTarget = normalizeSourcePath(direct); + if (directTarget) return directTarget; + + const name = String(toolName || ""); + if (name !== "Bash" && !/(?:^|__)shell(?:_|$)/i.test(name) && name !== "commandExecution") { + return null; + } + const command = String(input?.command ?? input?.commandText ?? ""); + const match = command.match(SOURCE_READ_COMMAND); + return normalizeSourcePath(match?.[1] || match?.[2] || match?.[3]); +} + +function resolveReference(currentTarget, reference) { + const raw = String(reference || "").trim().replace(/\\/g, "/"); + if (!raw || !SOURCE_PATH.test(raw)) return null; + if (raw.startsWith("./") || raw.startsWith("../")) { + return normalizeSourcePath(posix.join(posix.dirname(currentTarget), raw)); + } + return normalizeSourcePath(raw); +} + +function referencedSourceTargets(response, currentTarget) { + if (!currentTarget) return []; + const text = asText(response); + const found = new Set(); + const patterns = [ + /\b(?:from|require\s*\(|import\s*\()\s*["']([^"']+)["']/g, + /["']((?:\.\.?\/|[A-Za-z0-9_.-]+\/)[^"']+\.[A-Za-z0-9]+)["']/g, + ]; + for (const pattern of patterns) { + for (const match of text.matchAll(pattern)) { + const target = resolveReference(currentTarget, match[1]); + if (target) found.add(target); + } + } + return [...found].sort(); +} + +function positiveInteger(value, fallback) { + return Number.isInteger(value) && value > 0 ? value : fallback; +} + +function nonNegativeInteger(value) { + return Number.isInteger(value) && value >= 0 ? value : 0; +} + +export function createProgressWatchdog(options = {}) { + const base = createBaseProgressWatchdog(options); + const persistedCreditLimit = positiveInteger( + options.investigationCreditLimitPersisted, + DEFAULT_INVESTIGATION_CREDIT_LIMIT, + ); + const creditLimit = positiveInteger(options.investigationCreditLimit, persistedCreditLimit); + let creditGeneration = Number.isInteger(options.investigationCreditGeneration) + ? options.investigationCreditGeneration + : base.snapshot().stateGeneration; + let creditsUsed = nonNegativeInteger(options.investigationCreditsUsed); + let creditedSinceHydration = 0; + let referencedTargets = new Set( + Array.isArray(options.investigationReferencedTargets) + ? options.investigationReferencedTargets.map(normalizeSourcePath).filter(Boolean) + : [], + ); + let pendingCreditFingerprint = null; + + function syncGeneration() { + const generation = base.snapshot().stateGeneration; + if (generation !== creditGeneration) { + creditGeneration = generation; + creditsUsed = 0; + referencedTargets = new Set(); + pendingCreditFingerprint = null; + } + return generation; + } + + function decideRead(options = {}) { + const decision = base.decideRead(options); + syncGeneration(); + if (options.record === false) { + pendingCreditFingerprint = null; + if (decision.action === "allow" && options.volatility !== "volatile") { + const target = sourceReadTarget(options.toolName, options.input); + if (target && referencedTargets.has(target) && creditsUsed < creditLimit) { + pendingCreditFingerprint = decision.fingerprint; + } + } + } + return decision; + } + + function prepareEvidenceAttempt({ toolName, input, volatility = "stable" } = {}) { + syncGeneration(); + pendingCreditFingerprint = null; + if (volatility === "volatile" || creditsUsed >= creditLimit) return false; + const target = sourceReadTarget(toolName, input); + if (!target || !referencedTargets.has(target)) return false; + pendingCreditFingerprint = `prepared:${target}`; + return true; + } + + function chargeEvidenceAttempt() { + syncGeneration(); + if (pendingCreditFingerprint && creditsUsed < creditLimit) { + creditsUsed += 1; + creditedSinceHydration += 1; + pendingCreditFingerprint = null; + const current = base.snapshot(); + return { + action: "allow", + reason: "dependency_following_investigation_progress", + investigationProgress: true, + investigationCreditsUsed: creditsUsed, + investigationCreditLimit: creditLimit, + consecutiveEvidenceAttempts: current.consecutiveEvidenceAttempts, + totalEvidenceAttempts: current.totalEvidenceAttempts + creditedSinceHydration, + }; + } + const decision = base.chargeEvidenceAttempt(); + return { + ...decision, + totalEvidenceAttempts: decision.totalEvidenceAttempts + creditedSinceHydration, + investigationCreditsUsed: creditsUsed, + investigationCreditLimit: creditLimit, + }; + } + + function recordEvidenceResult({ toolName, input, volatility = "stable", response } = {}) { + syncGeneration(); + if (volatility === "volatile") { + referencedTargets = new Set(); + return []; + } + const target = sourceReadTarget(toolName, input); + referencedTargets = new Set(referencedSourceTargets(response, target)); + return [...referencedTargets]; + } + + function snapshot() { + syncGeneration(); + const current = base.snapshot(); + return { + ...current, + totalEvidenceAttempts: current.totalEvidenceAttempts + creditedSinceHydration, + investigationCreditGeneration: creditGeneration, + investigationCreditsUsed: creditsUsed, + investigationCreditLimit: creditLimit, + investigationCreditLimitPersisted: creditLimit, + investigationReferencedTargets: [...referencedTargets].sort(), + }; + } + + return { + ...base, + decideRead, + prepareEvidenceAttempt, + chargeEvidenceAttempt, + recordEvidenceResult, + snapshot, + }; +} diff --git a/tests/unit/watchdog-investigation-progress.test.mjs b/tests/unit/watchdog-investigation-progress.test.mjs new file mode 100644 index 00000000..cc991081 --- /dev/null +++ b/tests/unit/watchdog-investigation-progress.test.mjs @@ -0,0 +1,298 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { evaluateCodexHook } from "../../scripts/lib/codex-watchdog-hook.mjs"; +import { observeCodexAppServerMessage } from "../../scripts/lib/codex-progress-watchdog.mjs"; +import { createProgressWatchdog } from "../../scripts/lib/watchdog-investigation-progress.mjs"; + +function preEvidence(state, command, now, options = {}) { + return evaluateCodexHook( + { + hook_event_name: "PreToolUse", + tool_name: "Bash", + tool_input: { command }, + }, + state, + { + now, + evidenceSoftLimit: 2, + evidenceHardLimit: 3, + ...options, + }, + ); +} + +function runEvidence(state, command, response, now, options = {}) { + const pre = preEvidence(state, command, now, options); + assert.equal(pre.output?.decision, undefined, pre.output?.reason); + return evaluateCodexHook( + { + hook_event_name: "PostToolUse", + tool_name: "Bash", + tool_input: { command }, + tool_response: response, + }, + pre.state, + { + now: now + 1, + evidenceSoftLimit: 2, + evidenceHardLimit: 3, + ...options, + }, + ).state; +} + +function runAppServerEvidence(watchdog, context, command, response, index) { + const common = { + threadId: "thr-investigation", + turnId: "turn-investigation", + }; + const started = observeCodexAppServerMessage( + watchdog, + { + method: "item/started", + params: { + ...common, + item: { + id: `cmd-${index}`, + type: "commandExecution", + command, + status: "inProgress", + }, + }, + }, + context, + ); + assert.equal(started.interrupt, undefined, started.decision?.reason); + observeCodexAppServerMessage( + watchdog, + { + method: "item/completed", + params: { + ...common, + item: { + id: `cmd-${index}`, + type: "commandExecution", + command, + status: "completed", + exitCode: 0, + aggregatedOutput: response, + }, + }, + }, + context, + ); +} + +test("dependency-following source reads can reach one complete RED regression before execution", () => { + let state = {}; + state = runEvidence( + state, + "Get-Content src/authority-entrypoint.mjs", + 'import { resolveStoredCredential } from "./stored-credential-resolver.mjs";', + 1_000, + ); + state = runEvidence( + state, + "Get-Content src/stored-credential-resolver.mjs", + 'import { refreshManagedToken } from "./managed-token-refresh.mjs";', + 2_000, + ); + state = runEvidence( + state, + "Get-Content src/managed-token-refresh.mjs", + 'See the existing retry contract in "../tests/stored-account-retry.test.mjs".', + 3_000, + ); + state = runEvidence( + state, + "Get-Content tests/stored-account-retry.test.mjs", + "test(\"stored account refreshes once after 401\", () => {});", + 4_000, + ); + + assert.equal(state.watchdog.totalEvidenceAttempts, 4); + assert.equal(state.watchdog.investigationCreditsUsed, 3); +}); + +test("unrelated source reads still exhaust the ordinary consecutive evidence budget", () => { + let state = {}; + state = runEvidence(state, "Get-Content src/a.mjs", "export const a = 1;", 1_000); + state = runEvidence(state, "Get-Content src/b.mjs", "export const b = 1;", 2_000); + + const blocked = preEvidence(state, "Get-Content src/c.mjs", 3_000); + assert.equal(blocked.output?.decision, "block"); + assert.match(blocked.output?.reason || "", /Evidence exploration budget exhausted/); +}); + +test("assistant prose cannot manufacture dependency-following investigation credit", () => { + let state = runEvidence( + {}, + "Get-Content src/a.mjs", + "export const a = 1;", + 1_000, + { evidenceSoftLimit: 1, evidenceHardLimit: 2 }, + ); + state = evaluateCodexHook( + { + hook_event_name: "Stop", + last_assistant_message: 'The next dependency is "src/b.mjs".', + }, + state, + { now: 1_500, evidenceSoftLimit: 1, evidenceHardLimit: 2 }, + ).state; + + const blocked = preEvidence( + state, + "Get-Content src/b.mjs", + 2_000, + { evidenceSoftLimit: 1, evidenceHardLimit: 2 }, + ); + assert.equal(blocked.output?.decision, "block"); +}); + +test("dependency-following investigation credit is capped per state generation", () => { + const options = { investigationCreditLimit: 2 }; + let state = {}; + state = runEvidence( + state, + "Get-Content src/a.mjs", + 'export { b } from "./b.mjs";', + 1_000, + options, + ); + state = runEvidence( + state, + "Get-Content src/b.mjs", + 'export { c } from "./c.mjs";', + 2_000, + options, + ); + state = runEvidence( + state, + "Get-Content src/c.mjs", + 'export { d } from "./d.mjs";', + 3_000, + options, + ); + state = runEvidence( + state, + "Get-Content src/d.mjs", + 'export { e } from "./e.mjs";', + 4_000, + options, + ); + + const blocked = preEvidence(state, "Get-Content src/e.mjs", 5_000, options); + assert.equal(blocked.output?.decision, "block"); + assert.equal(state.watchdog.investigationCreditsUsed, 2); +}); + +test("linked duplicate stable reads remain blocked before investigation credit", () => { + let state = runEvidence( + {}, + "Get-Content src/a.mjs", + 'export { b } from "./b.mjs";', + 1_000, + ); + state = runEvidence( + state, + "Get-Content src/b.mjs", + 'export { c } from "./c.mjs";', + 2_000, + ); + + const duplicate = preEvidence(state, "Get-Content src/b.mjs", 3_000); + assert.equal(duplicate.output?.decision, "block"); + assert.match(duplicate.output?.reason || "", /Duplicate read blocked/); +}); + +test("Codex App Server receives the same bounded dependency-following evidence credit", () => { + const watchdog = createProgressWatchdog({ + evidenceSoftLimit: 2, + evidenceHardLimit: 3, + }); + const context = { interruptedTurns: new Set() }; + + runAppServerEvidence( + watchdog, + context, + "Get-Content src/authority-entrypoint.mjs", + 'import { resolveStoredCredential } from "./stored-credential-resolver.mjs";', + 1, + ); + runAppServerEvidence( + watchdog, + context, + "Get-Content src/stored-credential-resolver.mjs", + 'import { refreshManagedToken } from "./managed-token-refresh.mjs";', + 2, + ); + runAppServerEvidence( + watchdog, + context, + "Get-Content src/managed-token-refresh.mjs", + 'See "../tests/stored-account-retry.test.mjs".', + 3, + ); + runAppServerEvidence( + watchdog, + context, + "Get-Content tests/stored-account-retry.test.mjs", + "test(\"stored account refreshes once after 401\", () => {});", + 4, + ); + + assert.equal(watchdog.snapshot().totalEvidenceAttempts, 4); + assert.equal(watchdog.snapshot().investigationCreditsUsed, 3); +}); + +test("volatile evidence never receives dependency-following investigation credit", () => { + const watchdog = createProgressWatchdog({ evidenceSoftLimit: 1, evidenceHardLimit: 2 }); + watchdog.chargeEvidenceAttempt(); + watchdog.recordEvidenceResult({ + toolName: "commandExecution", + input: { command: "Get-Content src/a.mjs" }, + volatility: "stable", + response: 'export { b } from "./b.mjs";', + }); + + assert.equal( + watchdog.prepareEvidenceAttempt({ + toolName: "commandExecution", + input: { command: "Get-Content src/b.mjs" }, + volatility: "volatile", + }), + false, + ); + const blocked = watchdog.chargeEvidenceAttempt(); + assert.equal(blocked.action, "block"); + assert.equal(blocked.reason, "evidence_budget_exhausted"); + assert.equal(watchdog.snapshot().investigationCreditsUsed, 0); +}); + +test("state progress clears dependency targets and resets the investigation credit generation", () => { + const watchdog = createProgressWatchdog({ evidenceSoftLimit: 2, evidenceHardLimit: 3 }); + watchdog.chargeEvidenceAttempt(); + watchdog.recordEvidenceResult({ + toolName: "commandExecution", + input: { command: "Get-Content src/a.mjs" }, + volatility: "stable", + response: 'export { b } from "./b.mjs";', + }); + assert.equal( + watchdog.prepareEvidenceAttempt({ + toolName: "commandExecution", + input: { command: "Get-Content src/b.mjs" }, + volatility: "stable", + }), + true, + ); + assert.equal(watchdog.chargeEvidenceAttempt().investigationProgress, true); + assert.equal(watchdog.snapshot().investigationCreditsUsed, 1); + + watchdog.recordStateProgress("test_state_progress"); + const reset = watchdog.snapshot(); + assert.equal(reset.investigationCreditsUsed, 0); + assert.deepEqual(reset.investigationReferencedTargets, []); +});