From df3b831f37801ee25616a85f69b4c1126ef4c26b Mon Sep 17 00:00:00 2001 From: Georgy Butaev <41178744+g-but@users.noreply.github.com> Date: Sat, 29 Aug 2026 07:09:39 +0200 Subject: [PATCH] fix(cat): the outcome gate reported a cause it had never tested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly gate failed with "assistant messages exist in the window but 0 Cat actions were logged — the cat_action_log write path looks dead (check RLS policies and the executor)" while cat_action_log held TEN rows for that window, the newest from 2026-08-25. The write path was alive. The gate had never looked at it. `proposed` counts only rows that are BOTH status=completed AND a create_* action. In the window there were none — the single create_* was `create_cause`, DENIED. Three different worlds produce proposed === 0: 1. nothing is written at all → the audit trail is dead. Page. 2. writes happen, no completed creates → nothing creatable was asked for, or every attempt was denied. Not an outage. 3. no assistant activity at all → nothing to grade. The old predicate could not tell 1 from 2, so it asserted the diagnosis for 1 whenever it saw either. It now fetches the window's rows with no status or action filter and fails only on the shape it was built for: the log EMPTY while the Cat is talking. Otherwise it reports what it found — including denials, which are a real signal worth seeing and were being swallowed. This matters beyond the noise: a tripwire that cries every night is one nobody believes on the night it is right, which is the exact failure it exists to catch. The predicate is now a pure function and main() only runs on direct execution, so the decision can be tested without a database — it stayed wrong because nothing could reach it. Seven cases added, run in a child process against the real module (matching eval-cat-retry's pattern). Proven by mutation: restoring the old predicate turns three of them red, and the artifact was verified byte-identical afterwards. Live against production: exits 0 with "the write path is alive — 10 action(s) logged in window (denied 4, completed 5, failed 1), just none of them a completed create_*." Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AqzRcMP1uJzd7Tav5fNxQz --- .../scripts/eval-cat-outcomes-gate.test.ts | 108 +++++++++++++++++ scripts/eval-cat-outcomes.mjs | 113 +++++++++++++++--- 2 files changed, 205 insertions(+), 16 deletions(-) create mode 100644 __tests__/unit/scripts/eval-cat-outcomes-gate.test.ts diff --git a/__tests__/unit/scripts/eval-cat-outcomes-gate.test.ts b/__tests__/unit/scripts/eval-cat-outcomes-gate.test.ts new file mode 100644 index 000000000..ce2d506cf --- /dev/null +++ b/__tests__/unit/scripts/eval-cat-outcomes-gate.test.ts @@ -0,0 +1,108 @@ +/** + * The nightly Cat outcome gate paged every night with + * + * "GATE FAILED: assistant messages exist in the window but 0 Cat actions were + * logged — the cat_action_log write path looks dead" + * + * while cat_action_log held ten rows for that window, the newest from + * 2026-08-25. The write path was alive; the gate had never looked at it. Its + * predicate counted only rows that are BOTH status=completed AND a create_* + * action, found none, and reported a cause it had not tested — the single + * create_* in the window was `create_cause`, DENIED. + * + * That mattered twice over: it woke a human nightly for a non-event, and a + * tripwire that cries every night is one nobody believes on the night it is + * right — which is exactly the failure this gate exists to catch. + * + * Run in a child process against the real module, matching eval-cat-retry's + * pattern: the thing worth protecting is the behaviour of the shipped file, + * not the presence of a keyword in its source. + */ + +import { execFileSync } from 'child_process'; +import path from 'path'; + +const MODULE_URL = `file://${path.join(process.cwd(), 'scripts/eval-cat-outcomes.mjs')}`; + +function verdict(input: Record): { fail: boolean; message: string } { + const script = ` + const { gateVerdict } = await import('${MODULE_URL}'); + console.log(JSON.stringify(gateVerdict(${JSON.stringify(input)}))); + `; + const out = execFileSync(process.execPath, ['--input-type=module', '-e', script], { + cwd: process.cwd(), + encoding: 'utf8', + timeout: 30_000, + env: { ...process.env, NODE_OPTIONS: '' }, + }); + return JSON.parse(out.trim().split('\n').pop()!); +} + +describe('eval-cat-outcomes gate predicate', () => { + it('importing the module neither runs the report nor exits', () => { + // main() is guarded to direct execution. Without that, the predicate could + // not be reached from a test at all — which is why it stayed wrong. + const out = execFileSync( + process.execPath, + ['--input-type=module', '-e', `await import('${MODULE_URL}'); console.log('imported');`], + { cwd: process.cwd(), encoding: 'utf8', timeout: 30_000, env: { ...process.env, NODE_OPTIONS: '' } } + ); + expect(out).toContain('imported'); + expect(out).not.toContain('Cat outcome funnel'); + }); + + it('passes when completed creates exist', () => { + expect(verdict({ proposed: 3, hasAssistantActivity: true, logRows: [] }).fail).toBe(false); + }); + + it('passes when the Cat was never active in the window', () => { + const v = verdict({ proposed: 0, hasAssistantActivity: false, logRows: [] }); + expect(v.fail).toBe(false); + expect(v.message).toMatch(/nothing to grade/); + }); + + it('FAILS only when the log is EMPTY while the Cat is talking — the real bug', () => { + const v = verdict({ proposed: 0, hasAssistantActivity: true, logRows: [] }); + expect(v.fail).toBe(true); + expect(v.message).toMatch(/EMPTY/); + }); + + it('does NOT fail when the log has rows but none is a completed create_*', () => { + // The exact 2026-08-29 window that paged: writes happening, no completed create. + const v = verdict({ + proposed: 0, + hasAssistantActivity: true, + logRows: [ + { action_id: 'publish_entity', status: 'denied' }, + { action_id: 'update_profile', status: 'completed' }, + { action_id: 'create_cause', status: 'denied' }, + { action_id: 'add_wallet', status: 'denied' }, + { action_id: 'send_payment', status: 'failed' }, + ], + }); + expect(v.fail).toBe(false); + expect(v.message).toMatch(/write path is alive/); + }); + + it('surfaces denials as an observation rather than swallowing them', () => { + const v = verdict({ + proposed: 0, + hasAssistantActivity: true, + logRows: [ + { action_id: 'create_cause', status: 'denied' }, + { action_id: 'publish_entity', status: 'denied' }, + ], + }); + expect(v.fail).toBe(false); + expect(v.message).toMatch(/2 were DENIED/); + }); + + it('a single row of any kind disproves "nothing is written"', () => { + const v = verdict({ + proposed: 0, + hasAssistantActivity: true, + logRows: [{ action_id: 'forget_memories', status: 'completed' }], + }); + expect(v.fail).toBe(false); + }); +}); diff --git a/scripts/eval-cat-outcomes.mjs b/scripts/eval-cat-outcomes.mjs index fdd12a195..28a8c3c89 100644 --- a/scripts/eval-cat-outcomes.mjs +++ b/scripts/eval-cat-outcomes.mjs @@ -16,7 +16,10 @@ * Also acts as the REGRESSION GUARD for the "audit trail silently dead" class * (cat_action_log had no INSERT RLS policy for months — every write failed * silently): with --gate, exits 1 when the window shows Cat assistant activity - * (cat_messages) but ZERO logged actions — the exact signature of that bug. + * (cat_messages) but the action log is EMPTY — the exact signature of that bug. + * NOT when the log has rows and merely no completed create_*: that is a funnel + * fact (nothing creatable was asked for, or every attempt was denied), and + * conflating the two made this gate fail nightly while the write path was fine. * * Usage: * node scripts/eval-cat-outcomes.mjs # report, exit 0 @@ -29,6 +32,7 @@ */ import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; // --------------------------------------------------------------------------- // Env @@ -53,9 +57,11 @@ const WINDOW_DAYS = Number(process.env.OUTCOME_WINDOW_DAYS || 30); const JSON_OUT = process.env.OUTCOME_JSON_OUT || null; const GATE = process.argv.includes('--gate'); -if (!SUPABASE_URL || !SERVICE_KEY) { - console.error('eval-cat-outcomes: missing SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY'); - process.exit(2); +function requireEnv() { + if (!SUPABASE_URL || !SERVICE_KEY) { + console.error('eval-cat-outcomes: missing SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY'); + process.exit(2); + } } /** @@ -92,6 +98,66 @@ async function rest(path) { return res.json(); } +// --------------------------------------------------------------------------- +// The gate +// --------------------------------------------------------------------------- + +/** + * Decide whether zero *completed create_** rows means the audit trail is dead. + * + * It usually does not. On 2026-08-29 this gate failed nightly with "the + * cat_action_log write path looks dead (check RLS policies and the executor)" + * while the log held ten rows for the window, the newest from 2026-08-25 — the + * write path was demonstrably alive. `proposed` counts only rows that are BOTH + * `status=completed` AND a `create_*` action, and in that window there were + * none: the single create_* was `create_cause`, DENIED. Three different worlds + * produce proposed === 0: + * + * 1. nothing is written at all → the audit trail is dead. Page. + * 2. writes happen, no completed creates → users asked for nothing creatable, + * or every attempt was denied. A + * funnel fact. Not an outage. + * 3. no assistant activity at all → nothing to grade. + * + * The old predicate could not see the difference between 1 and 2, so it + * asserted a cause it had never tested — and a tripwire that cries every night + * is one nobody believes on the night it is right. So the gate now fails only + * on the shape it was actually built for: the log is EMPTY while the Cat is + * talking. + */ +export function gateVerdict({ proposed, hasAssistantActivity, logRows }) { + if (proposed > 0) { + return { fail: false, message: `gate: ${proposed} completed create_* action(s) in window.` }; + } + if (!hasAssistantActivity) { + return { fail: false, message: 'gate: no assistant activity in window either — nothing to grade, pass.' }; + } + const rows = logRows || []; + if (rows.length === 0) { + return { + fail: true, + message: + 'GATE FAILED: assistant messages exist in the window but cat_action_log is EMPTY — ' + + 'the write path looks dead (check RLS policies and the executor).', + }; + } + const byStatus = {}; + for (const r of rows) { + byStatus[r.status] = (byStatus[r.status] || 0) + 1; + } + const summary = Object.entries(byStatus) + .map(([k, v]) => `${k} ${v}`) + .join(', '); + const denied = byStatus.denied || 0; + return { + fail: false, + message: + `gate: the write path is alive — ${rows.length} action(s) logged in window (${summary}), ` + + `just none of them a completed create_*. Not an outage.` + + (denied > 0 ? ` NOTE: ${denied} were DENIED — worth a look if that is unexpected.` : ''), + }; +} + // --------------------------------------------------------------------------- // Funnel // --------------------------------------------------------------------------- @@ -182,24 +248,39 @@ async function main() { console.log(` report → ${JSON_OUT}`); } - // Dead-audit-trail gate: Cat talked but nothing was ever logged → the write - // path is broken again (RLS, schema drift, refactor). Fail loudly. + // Dead-audit-trail gate — see gateVerdict for why it needs three inputs. if (GATE && report.proposed === 0) { const msgs = await rest( `cat_messages?select=id&role=eq.assistant&created_at=gte.${since}&limit=1` ); - if (msgs.length > 0) { - console.error( - `GATE FAILED: assistant messages exist in the window but 0 Cat actions were logged — ` + - `the cat_action_log write path looks dead (check RLS policies and the executor).` - ); + // The rows the gate must look at are ALL of them, with no status or action + // filter: "nothing is written at all" is the only shape that means the + // write path is dead. Fetching this is the whole fix. + const logAny = await rest( + `cat_action_log?select=action_id,status&created_at=gte.${since}&order=created_at.desc&limit=200` + ); + const verdict = gateVerdict({ + proposed: report.proposed, + hasAssistantActivity: msgs.length > 0, + logRows: logAny, + }); + if (verdict.fail) { + console.error(verdict.message); process.exit(1); } - console.log(' gate: no assistant activity in window either — nothing to grade, pass.'); + console.log(` ${verdict.message}`); } } -main().catch(err => { - console.error(`eval-cat-outcomes: ${err.message}`); - process.exit(2); -}); +// Run only when executed directly. Importing this module (the unit test does) +// must not hit the network or exit the process — that is what kept the gate's +// own predicate untestable while it was wrong. +const invokedDirectly = + process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedDirectly) { + requireEnv(); + main().catch(err => { + console.error(`eval-cat-outcomes: ${err.message}`); + process.exit(2); + }); +}