From cafb9b4263bb7debb94197dc897bcb4a12d7f25f Mon Sep 17 00:00:00 2001 From: Zenflow Date: Thu, 9 Jul 2026 18:06:31 -0700 Subject: [PATCH 1/5] Pass action expectations through desktop chat tasks --- docs/AURA_EXECUTION_TRACKER.md | 54 ++++++++++++++++++++++++++ interface/routes/chat.py | 13 +++++++ tests/test_server_conversation_lane.py | 13 +++++++ 3 files changed, 80 insertions(+) diff --git a/docs/AURA_EXECUTION_TRACKER.md b/docs/AURA_EXECUTION_TRACKER.md index f3f5d4b5a..18d1b8aee 100644 --- a/docs/AURA_EXECUTION_TRACKER.md +++ b/docs/AURA_EXECUTION_TRACKER.md @@ -8605,3 +8605,57 @@ Remaining work after this checkpoint: planning boundaries. - Add receipt/memory causal wiring for expectation failures. - Commit and push this checkpoint. + +## Checkpoint 2026-07-09-09: Chat Desktop Objective Expectation Propagation + +Scope: + +- Added `_desktop_task_action_expectation(...)` to the chat desktop objective + lane so live user requests that become `desktop_task` calls automatically + carry action-depth expectations. +- `_execute_desktop_objective_from_chat(...)` now passes the expectation in + both params and execution context. The central `CapabilityEngine` expectation + hook therefore sees the contract before returning success to chat. +- The current default expectation requires: + - `steps_requested` + - `steps_completed` + - `receipts` + - repair hint `rerun_desktop_task_with_effect_receipts` +- The existing chat verifier still performs the deeper step-level proof: + receipt shape, critical step success, `effect_verified=true`, non-empty + observable effect evidence, and rejection of receipt-id-only proof. + +Verification: + +- `python -m pytest tests/test_server_conversation_lane.py::test_api_chat_desktop_objective_requires_cognitive_planning tests/test_server_conversation_lane.py::test_chat_desktop_objective_uses_capability_engine_without_agency_wrapper tests/test_server_conversation_lane.py::test_chat_desktop_research_objective_does_not_enable_hidden_model_synthesis tests/test_server_conversation_lane.py::test_chat_desktop_objective_rejects_success_without_effect_receipts tests/test_capability_engine_policy_regressions.py::test_execute_with_retry_downgrades_shallow_action_expectation tests/test_capability_engine_policy_regressions.py::test_execute_with_retry_marks_missing_expectation_evidence_unverified -q` + -> `6 passed`. +- `python -m pytest tests/test_server_conversation_lane.py::test_api_chat_desktop_surface_plans_with_cognitive_engine_before_execution -q` + -> `1 passed`. +- `python -m pytest tests/test_server_conversation_lane.py -q -k desktop_objective` + -> `6 passed, 237 deselected`. +- `python -m py_compile interface/routes/chat.py tests/test_server_conversation_lane.py core/capability_engine.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `python -m ruff check --select F,E9 interface/routes/chat.py tests/test_server_conversation_lane.py core/capability_engine.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `make production-gate` + -> passed; `/tmp/aura_production_readiness.json` has `passed=true`. +- `make enterprise-gate` + -> passed; `/tmp/aura_enterprise_gate.json` has `counts={}` and + `high_or_critical_count=0`. +- `git diff --check` + -> passed. + +Current closeout estimate: + +- Pass F action-depth enforcement now reaches the live chat -> desktop_task + path for visible desktop objectives. It is still intentionally explicit and + contract-driven; broader automatic expectation generation for non-desktop + skills remains open. + +Remaining work after this checkpoint: + +- Add durable receipt/memory wiring for expectation verdicts. +- Expand expectation generation beyond desktop objectives: web research, + file/memory actions, live skill API proofs, autonomous overt actions, and + chat follow-through. +- Commit and push this checkpoint. diff --git a/interface/routes/chat.py b/interface/routes/chat.py index de467ad08..a24bb7fa9 100644 --- a/interface/routes/chat.py +++ b/interface/routes/chat.py @@ -12956,6 +12956,16 @@ def _verified_desktop_task_result(result: dict[str, Any]) -> tuple[bool, str]: return True, "verified" +def _desktop_task_action_expectation(objective: str) -> dict[str, Any]: + return { + "objective": str(objective or "")[:500], + "acceptance_criteria": ["steps_requested", "steps_completed"], + "required_evidence": ["receipts"], + "repair_hint": "rerun_desktop_task_with_effect_receipts", + "allow_partial": True, + } + + def _desktop_objective_self_sufficient_without_cognitive_text(user_message: str) -> bool: """Whether desktop_task can honestly complete without a model-composed body. @@ -13121,6 +13131,7 @@ async def _execute_desktop_objective_from_chat( # explicit callers can still opt into model synthesis by invoking # desktop_task directly with allow_desktop_task_model_synthesis=True. allow_research_synthesis = False + action_expectation = _desktop_task_action_expectation(objective) desktop_params = { "objective": objective, "steps": [], @@ -13134,6 +13145,7 @@ async def _execute_desktop_objective_from_chat( "local_desktop_action": True, "verification_required": True, "predicted_outcome": "The requested visible desktop/file effect is verified after execution.", + "action_expectation": action_expectation, } result = await _execute_governed_live_skill( "desktop_task", @@ -13152,6 +13164,7 @@ async def _execute_desktop_objective_from_chat( "allow_desktop_task_model_synthesis": allow_research_synthesis, "desktop_task_document_body": str(cognitive_reply or "").strip(), "cognitive_reply": str(cognitive_reply or "").strip(), + "action_expectation": action_expectation, }, ) if not isinstance(result, dict): diff --git a/tests/test_server_conversation_lane.py b/tests/test_server_conversation_lane.py index c0eccecb6..9c5aec0f3 100644 --- a/tests/test_server_conversation_lane.py +++ b/tests/test_server_conversation_lane.py @@ -3978,6 +3978,13 @@ def _live_proof_lane_status(): assert skill_calls[0]["params"]["allow_heuristic_desktop_plan"] is True assert skill_calls[0]["params"]["user_visible_desktop_action"] is True assert skill_calls[0]["params"]["verification_required"] is True + assert skill_calls[0]["params"]["action_expectation"] == { + "objective": skill_calls[0]["params"]["objective"], + "acceptance_criteria": ["steps_requested", "steps_completed"], + "required_evidence": ["receipts"], + "repair_hint": "rerun_desktop_task_with_effect_receipts", + "allow_partial": True, + } assert skill_calls[0]["objective"] == skill_calls[0]["params"]["objective"] assert skill_calls[0]["extra_context"] == { "origin": "desktop_ui", @@ -3992,6 +3999,7 @@ def _live_proof_lane_status(): "allow_desktop_task_model_synthesis": False, "desktop_task_document_body": skill_calls[0]["extra_context"]["cognitive_reply"], "cognitive_reply": skill_calls[0]["extra_context"]["cognitive_reply"], + "action_expectation": skill_calls[0]["params"]["action_expectation"], } assert "Timestamped Aura summary from CognitiveEngine." in skill_calls[0]["extra_context"]["cognitive_reply"] assert completed_exchanges @@ -4073,6 +4081,8 @@ async def run(self, *_args, **_kwargs): assert calls[0]["context"]["user_explicitly_authorized"] is True assert calls[0]["context"]["allow_heuristic_desktop_plan"] is True assert calls[0]["context"]["allow_desktop_task_model_synthesis"] is False + assert calls[0]["params"]["action_expectation"]["required_evidence"] == ["receipts"] + assert calls[0]["context"]["action_expectation"] == calls[0]["params"]["action_expectation"] @pytest.mark.asyncio @@ -4121,6 +4131,9 @@ async def execute(self, skill_name, params, context=None): assert calls and calls[0]["skill_name"] == "desktop_task" assert calls[0]["context"]["route"] == "chat.desktop_objective" assert calls[0]["context"]["allow_desktop_task_model_synthesis"] is False + assert calls[0]["context"]["action_expectation"]["repair_hint"] == ( + "rerun_desktop_task_with_effect_receipts" + ) @pytest.mark.asyncio From fade7e5f2b245104a8cfa628c483c63fe6302492 Mon Sep 17 00:00:00 2001 From: Zenflow Date: Thu, 9 Jul 2026 18:13:50 -0700 Subject: [PATCH 2/5] Register Pass F maturity failure modes --- core/resilience/fault_taxonomy.py | 158 ++++++++++++++++ core/resilience/fmea_registry.py | 243 +++++++++++++++++++++++++ docs/AURA_EXECUTION_TRACKER.md | 76 ++++++++ docs/runbooks/README.md | 1 + docs/runbooks/pass-f-maturity-risks.md | 59 ++++++ tests/test_reliability_hardening.py | 47 +++++ 6 files changed, 584 insertions(+) create mode 100644 docs/runbooks/pass-f-maturity-risks.md diff --git a/core/resilience/fault_taxonomy.py b/core/resilience/fault_taxonomy.py index 211774021..a93b838cb 100644 --- a/core/resilience/fault_taxonomy.py +++ b/core/resilience/fault_taxonomy.py @@ -567,6 +567,164 @@ def _register_builtin_faults(self) -> None: recovery=RecoveryStrategy.GRACEFUL_DEGRADATION, mttr_seconds=0, blast_radius="Logged and monitored; function continues (log+continue mode)", ), + FaultDefinition( + fault_id="PASSF-ACTION-SHALLOW-SUCCESS", + name="Shallow action reports success", + description="An action fires and returns a technically true success " + "without satisfying the user's implied acceptance " + "criteria or preserving effect evidence.", + domain=FaultDomain.TOOL_EXECUTION, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.PROBABLE, + detection=DetectionDifficulty.LOW, + recovery=RecoveryStrategy.RETRY_WITH_BACKOFF, + mttr_seconds=30, + blast_radius="User-facing task appears complete while the useful " + "work is missing or too shallow to rely on.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-FALSE-HEALTH", + name="False health/readiness signal", + description="A readiness, liveness, or proof-health path reports " + "green while the live user path remains blocked.", + domain=FaultDomain.OBSERVABILITY, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.PROBABLE, + detection=DetectionDifficulty.LOW, + recovery=RecoveryStrategy.QUARANTINE, + mttr_seconds=60, + blast_radius="Operators trust a green signal that does not match " + "actual demo or daily-use readiness.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-RESOURCE-SPAWN-LOOP", + name="Resource spawn loop", + description="Model, memory, or worker orchestration repeatedly " + "spawns work under pressure instead of admitting, " + "backing off, or degrading explicitly.", + domain=FaultDomain.RESOURCE, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.OCCASIONAL, + detection=DetectionDifficulty.MODERATE, + recovery=RecoveryStrategy.CIRCUIT_BREAKER, + mttr_seconds=45, + blast_radius="GPU/RAM pressure cascades into stalled requests, " + "wedged workers, or unreliable boot.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-DESKTOP-PERMISSION-DRIFT", + name="Desktop permission drift", + description="Desktop, Chrome, accessibility, camera, microphone, " + "or browser-control permissions drift after boot and " + "surface only as shallow action failure.", + domain=FaultDomain.TOOL_EXECUTION, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.OCCASIONAL, + detection=DetectionDifficulty.MODERATE, + recovery=RecoveryStrategy.GRACEFUL_DEGRADATION, + mttr_seconds=120, + blast_radius="Visible computer-use workflows cannot be demoed or " + "completed despite nominal capability registration.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-REPAIR-STORM", + name="Self-repair storm", + description="Autonomy or self-repair loops continue patching, " + "retrying, or re-planning without cooldown, budget, " + "causal progress, or operator-visible stop condition.", + domain=FaultDomain.AGENCY, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.REMOTE, + detection=DetectionDifficulty.LOW, + recovery=RecoveryStrategy.CIRCUIT_BREAKER, + mttr_seconds=60, + blast_radius="Repair behavior consumes resources and can make " + "the original fault harder to diagnose.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-STALE-OBLIGATION", + name="Stale obligation blocks current work", + description="Memory, prompt, or task-state residue from an older " + "objective keeps steering unrelated current work.", + domain=FaultDomain.MEMORY, + severity=FaultSeverity.MARGINAL, + probability=FaultProbability.PROBABLE, + detection=DetectionDifficulty.LOW, + recovery=RecoveryStrategy.AUTOMATIC_FALLBACK, + mttr_seconds=30, + blast_radius="Aura acts federated and distracted instead of " + "unified around the user's current objective.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-NEURAL-STREAM-FLOOD", + name="Neural stream flood hides state", + description="High-volume internal streams, events, or logs drown " + "out the actionable user-visible state needed for " + "debugging and daily operation.", + domain=FaultDomain.OBSERVABILITY, + severity=FaultSeverity.MARGINAL, + probability=FaultProbability.PROBABLE, + detection=DetectionDifficulty.MODERATE, + recovery=RecoveryStrategy.GRACEFUL_DEGRADATION, + mttr_seconds=30, + blast_radius="Operators see noise rather than crisp state, " + "blockers, receipts, and next actions.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-VISIBLE-WEB-PROOF-ACCESS", + name="Visible web proof inaccessible", + description="Visible Chrome or web-interlocutor proof depends on " + "a browser profile, extension, or session that is not " + "available to the active runtime.", + domain=FaultDomain.TOOL_EXECUTION, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.OCCASIONAL, + detection=DetectionDifficulty.HIGH, + recovery=RecoveryStrategy.MANUAL_INTERVENTION, + mttr_seconds=300, + blast_radius="External web proof and demo-critical browser " + "workflows remain honestly blocked.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-PROOF-ARTIFACT-CONTAMINATION", + name="Proof artifact contamination", + description="A proof, certification, or report consumes stale, " + "fixture-backed, hardcoded, or cross-run artifact " + "data as if it were fresh live evidence.", + domain=FaultDomain.OBSERVABILITY, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.REMOTE, + detection=DetectionDifficulty.LOW, + recovery=RecoveryStrategy.QUARANTINE, + mttr_seconds=120, + blast_radius="Certification can overstate maturity and hide " + "runtime regressions until live demo.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), + FaultDefinition( + fault_id="PASSF-SEMANTIC-REVIEW-GAP", + name="Semantic review gap", + description="Mechanical closeout, line counting, or path hashing " + "is mistaken for semantic review of code behavior, " + "runtime contracts, and user-facing obligations.", + domain=FaultDomain.GOVERNANCE, + severity=FaultSeverity.CRITICAL, + probability=FaultProbability.PROBABLE, + detection=DetectionDifficulty.LOW, + recovery=RecoveryStrategy.MANUAL_INTERVENTION, + mttr_seconds=600, + blast_radius="Remaining code debt is hidden behind a green " + "mechanical audit surface.", + runbook="docs/runbooks/pass-f-maturity-risks.md", + ), ] for defn in builtins: self._definitions[defn.fault_id] = defn diff --git a/core/resilience/fmea_registry.py b/core/resilience/fmea_registry.py index 91ef66f53..d992ffc25 100644 --- a/core/resilience/fmea_registry.py +++ b/core/resilience/fmea_registry.py @@ -121,6 +121,7 @@ def full_report(self) -> list[dict[str, Any]]: "blast_radius": defn.blast_radius, "runbook": defn.runbook, "mitigated": entry.is_mitigated(), + "mitigation_count": len(entry.mitigations), "mitigations": [ { "action_id": m.action_id, @@ -301,6 +302,248 @@ def _register_builtins(self) -> None: automated=True, implementation_path="core/resilience/contracts.py", verified=False), ], notes="Mitigation added during reliability hardening"), + "ACTION-CLAIM-MISMATCH": FMEAEntry( + fault_id="ACTION-CLAIM-MISMATCH", + mitigations=[ + MitigationAction( + "MIT-ACM-1", + "Action expectation contract downgrades successful " + "returns when effect evidence or acceptance criteria are missing", + automated=True, + implementation_path="core/runtime/skill_contract.py", + verified=True, + ), + MitigationAction( + "MIT-ACM-2", + "CapabilityEngine enforces explicit action expectations " + "before returning success to callers", + automated=True, + implementation_path="core/capability_engine.py", + verified=True, + ), + ], + notes="Grounding gap closed by Pass F action-depth contract", + ), + "PASSF-ACTION-SHALLOW-SUCCESS": FMEAEntry( + fault_id="PASSF-ACTION-SHALLOW-SUCCESS", + mitigations=[ + MitigationAction( + "MIT-PASSF-ACTION-1", + "ActionExpectation evaluates user-visible effect, " + "acceptance criteria, evidence, and repair hints before " + "a result can remain success_verified", + automated=True, + implementation_path="core/runtime/skill_contract.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-ACTION-2", + "Desktop chat objectives pass expectation contracts " + "through the live capability execution lane", + automated=True, + implementation_path="interface/routes/chat.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-FALSE-HEALTH": FMEAEntry( + fault_id="PASSF-FALSE-HEALTH", + mitigations=[ + MitigationAction( + "MIT-PASSF-HEALTH-1", + "Production readiness gate separates proof readiness, " + "chat readiness, and health blockers", + automated=True, + implementation_path="tools/aura_production_readiness_gate.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-HEALTH-2", + "Live boot proof reports explicit readiness blockers " + "instead of treating boot as full user-path proof", + automated=True, + implementation_path="tools/live_boot_proof.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-RESOURCE-SPAWN-LOOP": FMEAEntry( + fault_id="PASSF-RESOURCE-SPAWN-LOOP", + mitigations=[ + MitigationAction( + "MIT-PASSF-RESOURCE-1", + "Resource governor tracks pressure and applies automatic " + "degradation before unbounded work piles up", + automated=True, + implementation_path="core/resilience/resource_governor.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-RESOURCE-2", + "Memory governor can demote tiers under pressure", + automated=True, + implementation_path="core/resilience/memory_governor.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-DESKTOP-PERMISSION-DRIFT": FMEAEntry( + fault_id="PASSF-DESKTOP-PERMISSION-DRIFT", + mitigations=[ + MitigationAction( + "MIT-PASSF-DESKTOP-1", + "Chat desktop verifier rejects critical-step success " + "without observable effect receipts", + automated=True, + implementation_path="interface/routes/chat.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-DESKTOP-2", + "Browser/desktop actor lifecycle is covered by runtime " + "hardening runbooks and leakage tests", + automated=True, + implementation_path="tests/test_server_runtime_hardening.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-REPAIR-STORM": FMEAEntry( + fault_id="PASSF-REPAIR-STORM", + mitigations=[ + MitigationAction( + "MIT-PASSF-REPAIR-1", + "Will/governance receipts give repair actions an " + "auditable decision boundary", + automated=True, + implementation_path="core/governance/will.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-REPAIR-2", + "Self-repair runbook defines safe mitigation, rollback, " + "and postmortem requirements", + automated=False, + implementation_path="docs/runbooks/self-repair-failed.md", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-STALE-OBLIGATION": FMEAEntry( + fault_id="PASSF-STALE-OBLIGATION", + mitigations=[ + MitigationAction( + "MIT-PASSF-STALE-1", + "Grounded recall keeps remembered context tied to " + "evidence instead of letting stale assertions dominate", + automated=True, + implementation_path="core/conversation/grounded_recall.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-STALE-2", + "Remaining checkpoint contract keeps open obligations " + "explicit rather than implicit prompt residue", + automated=True, + implementation_path="tools/closeout/remaining_checkpoint_contract.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-NEURAL-STREAM-FLOOD": FMEAEntry( + fault_id="PASSF-NEURAL-STREAM-FLOOD", + mitigations=[ + MitigationAction( + "MIT-PASSF-STREAM-1", + "Diagnostics dashboard exposes summarized FMEA, high-risk, " + "and unmitigated state instead of raw stream volume", + automated=True, + implementation_path="core/resilience/diagnostics_dashboard.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-STREAM-2", + "Distributed tracing uses per-trace sampling decisions " + "to prevent incoherent observability floods", + automated=True, + implementation_path="core/observability/tracing.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-VISIBLE-WEB-PROOF-ACCESS": FMEAEntry( + fault_id="PASSF-VISIBLE-WEB-PROOF-ACCESS", + mitigations=[ + MitigationAction( + "MIT-PASSF-WEB-1", + "Web interlocutor proof path records browser-access " + "blockers instead of silently substituting weak proof", + automated=True, + implementation_path="tools/proof/run_web_interlocutor_live_proof.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-WEB-2", + "Live boot proof keeps visible browser proof separate " + "from core boot readiness", + automated=True, + implementation_path="tools/live_boot_proof.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-PROOF-ARTIFACT-CONTAMINATION": FMEAEntry( + fault_id="PASSF-PROOF-ARTIFACT-CONTAMINATION", + mitigations=[ + MitigationAction( + "MIT-PASSF-PROOF-1", + "Proof fabrication guard rejects hardcoded pass/fail " + "scores and fixture-backed victory claims", + automated=True, + implementation_path="tools/proof_fabrication_guard.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-PROOF-2", + "Proof step runner wraps each proof command with hard " + "timeout and evidence artifact metadata", + automated=True, + implementation_path="tools/run_proof_step.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), + "PASSF-SEMANTIC-REVIEW-GAP": FMEAEntry( + fault_id="PASSF-SEMANTIC-REVIEW-GAP", + mitigations=[ + MitigationAction( + "MIT-PASSF-SEMANTIC-1", + "Semantic review ledger records per-file behavioral " + "review evidence separate from mechanical hashing", + automated=True, + implementation_path="tools/closeout/semantic_review_ledger.py", + verified=True, + ), + MitigationAction( + "MIT-PASSF-SEMANTIC-2", + "Codebase closeout audit reports semantic review " + "incompleteness instead of closing on text counts alone", + automated=True, + implementation_path="tools/closeout/run_codebase_closeout_audit.py", + verified=True, + ), + ], + notes="Pass F structural maturity risk", + ), } for fid, entry in entries.items(): self._entries[fid] = entry diff --git a/docs/AURA_EXECUTION_TRACKER.md b/docs/AURA_EXECUTION_TRACKER.md index 18d1b8aee..cc2e2ba73 100644 --- a/docs/AURA_EXECUTION_TRACKER.md +++ b/docs/AURA_EXECUTION_TRACKER.md @@ -8659,3 +8659,79 @@ Remaining work after this checkpoint: file/memory actions, live skill API proofs, autonomous overt actions, and chat follow-through. - Commit and push this checkpoint. + +## Checkpoint 2026-07-09-10: Pass F Failure-Mode Registry + +Scope: + +- Promoted the Pass F maturity concerns into the production fault taxonomy and + FMEA registry instead of leaving them as tracker prose. +- Added ten explicit `PASSF-*` fault definitions covering: + - shallow action success + - false health/readiness signals + - resource spawn loops + - desktop/browser permission drift + - repair storms + - stale obligations + - neural/event stream floods + - visible web proof access blockers + - proof artifact contamination + - semantic review gaps +- Added FMEA mitigation rows for those ten risks and for the older + `ACTION-CLAIM-MISMATCH` grounding fault. Each mitigation points at an + existing implementation or runbook artifact so traceability tests can catch + stale claims. +- Added `docs/runbooks/pass-f-maturity-risks.md` and linked it from the runbook + index. +- Extended the reliability hardening tests so Pass F risks must stay + registered, mitigated, linked to the runbook, and visible in the high-risk RPN + report when applicable. +- Added `mitigation_count` to `FMEARegistry.full_report()` so downstream + diagnostics can show mitigation depth without re-counting nested rows. + +Verification: + +- `python -m pytest tests/test_reliability_hardening.py -q -k "pass_f or FMEA or Traceability"` + -> `8 passed, 89 deselected`. +- `python -m pytest tests/test_reliability_hardening.py -q` + -> `97 passed`. +- `python -m pytest tests/test_action_depth_honesty.py tests/test_capability_engine_policy_regressions.py::test_execute_with_retry_downgrades_shallow_action_expectation tests/test_capability_engine_policy_regressions.py::test_execute_with_retry_marks_missing_expectation_evidence_unverified -q` + -> `19 passed`. +- `python -m py_compile core/resilience/fault_taxonomy.py core/resilience/fmea_registry.py tests/test_reliability_hardening.py` + -> passed. +- `python -m ruff check --select F,E9 core/resilience/fault_taxonomy.py core/resilience/fmea_registry.py tests/test_reliability_hardening.py` + -> passed. +- FMEA registry probe: + - `total_fault_definitions=33` + - `total_fmea_entries=33` + - `missing_fmea_entries=0` + - `coverage_pct=100.0` + - `passf_count=10` + - high-risk Pass F IDs: `PASSF-STALE-OBLIGATION`, + `PASSF-NEURAL-STREAM-FLOOD`, `PASSF-ACTION-SHALLOW-SUCCESS`, + `PASSF-FALSE-HEALTH`, `PASSF-SEMANTIC-REVIEW-GAP` +- `make production-gate` + -> passed; `/tmp/aura_production_readiness.json` has `passed=true` and + `37` checks. +- `make enterprise-gate` + -> passed; `/tmp/aura_enterprise_gate.json` has `counts={}` and + `high_or_critical_count=0`. +- `git diff --check` + -> passed. + +Current closeout estimate: + +- Pass F now has a machine-readable risk spine. These risks show up through the + same taxonomy/FMEA/diagnostics surfaces as earlier production reliability + faults instead of relying on free-form notes. +- This is not a full closure of the risks. Five Pass F risks intentionally + remain high-RPN because the registry now acknowledges them honestly while the + deeper runtime work continues. + +Remaining work after this checkpoint: + +- Build durable receipt/memory feedback for expectation verdicts. +- Expand automatic expectation generation beyond desktop tasks. +- Start the declarative runtime control plane and resource admission layer so + high-risk `PASSF-RESOURCE-SPAWN-LOOP` becomes preventable, not just visible. +- Keep semantic review coverage moving through the closeout ledger. diff --git a/docs/runbooks/README.md b/docs/runbooks/README.md index b11a144e4..ca789874d 100644 --- a/docs/runbooks/README.md +++ b/docs/runbooks/README.md @@ -23,3 +23,4 @@ safe mitigation, unsafe mitigation, rollback, and verification. | Camera unavailable | [camera-unavailable.md](camera-unavailable.md) | | Microphone unavailable | [microphone-unavailable.md](microphone-unavailable.md) | | Movie mode broken | [movie-mode-broken.md](movie-mode-broken.md) | +| Pass F maturity risks | [pass-f-maturity-risks.md](pass-f-maturity-risks.md) | diff --git a/docs/runbooks/pass-f-maturity-risks.md b/docs/runbooks/pass-f-maturity-risks.md new file mode 100644 index 000000000..4d67c15ec --- /dev/null +++ b/docs/runbooks/pass-f-maturity-risks.md @@ -0,0 +1,59 @@ +# Runbook: Pass F Maturity Risks + +## Symptoms +- An action returns success but lacks durable effect evidence, acceptance criteria, + or a user-visible result that matches the request. +- Health, readiness, or proof status is green while chat, desktop, browser, or + visible demo paths remain blocked. +- Resource pressure causes repeated model, worker, repair, or planner loops + instead of admission control, backoff, or explicit degradation. +- Old objectives, memories, or proof artifacts keep steering current work. +- Internal event streams are noisy enough that blockers, receipts, and next + actions are hard to identify. + +## Diagnosis +- Run `python -m pytest tests/test_action_depth_honesty.py -q`. +- Run `python -m pytest tests/test_reliability_hardening.py -q -k "pass_f or FMEA or Traceability"`. +- Inspect `get_fmea_registry().faults_above_rpn(30)` and confirm Pass F entries + remain visible. +- Compare live-path readiness to proof readiness before treating a green health + result as demo-ready. +- Check the latest closeout tracker entry for explicit blockers, not just + mechanical codebase hash or line-count results. + +## Safe Mitigation +- Downgrade shallow successes to `success_unverified`, `partial_success`, or + `failed_recoverable` and rerun with effect receipts. +- Quarantine contaminated proof artifacts and rerun the proof step through + `tools/run_proof_step.py`. +- Prefer admission control, resource governor degradation, and circuit breakers + over spawning more work under pressure. +- Surface browser/desktop permission blockers explicitly and do not substitute + fixture proof for visible proof. +- Record semantic review gaps in the closeout ledger instead of treating the + mechanical audit as completion. + +## Unsafe Mitigation +- Marking a task complete because a tool fired once. +- Treating `proof_readiness_healthy` as chat or browser demo readiness. +- Reusing prior proof artifacts without fresh run metadata. +- Suppressing noisy streams without preserving the actionable state summary. + +## Rollback +- Revert the change that introduced the shallow success, proof contamination, + or loop, then rerun the focused test named in Diagnosis. +- If rollback is not local to one patch, restore the last pushed checkpoint and + repeat production and enterprise gates before continuing closeout work. + +## Verification +- `python -m pytest tests/test_reliability_hardening.py -q -k "pass_f or Traceability"` +- `python -m pytest tests/test_action_depth_honesty.py -q` +- `make production-gate` +- `make enterprise-gate` + +## Postmortem Checklist +- Add or update the FMEA row, mitigation path, and runbook when a new maturity + risk is found. +- Add a regression test that proves the failure is detected before user impact. +- Update `docs/AURA_EXECUTION_TRACKER.md` with the blocker, mitigation, and + validation evidence. diff --git a/tests/test_reliability_hardening.py b/tests/test_reliability_hardening.py index e024c6379..991ff4f4c 100644 --- a/tests/test_reliability_hardening.py +++ b/tests/test_reliability_hardening.py @@ -100,6 +100,29 @@ def test_status(self): assert status["total_faults"] == 1 assert "definitions_count" in status + def test_pass_f_maturity_faults_are_registered(self): + from core.resilience.fault_taxonomy import FaultRegistry + + reg = FaultRegistry() + required = { + "PASSF-ACTION-SHALLOW-SUCCESS", + "PASSF-FALSE-HEALTH", + "PASSF-RESOURCE-SPAWN-LOOP", + "PASSF-DESKTOP-PERMISSION-DRIFT", + "PASSF-REPAIR-STORM", + "PASSF-STALE-OBLIGATION", + "PASSF-NEURAL-STREAM-FLOOD", + "PASSF-VISIBLE-WEB-PROOF-ACCESS", + "PASSF-PROOF-ARTIFACT-CONTAMINATION", + "PASSF-SEMANTIC-REVIEW-GAP", + } + definitions = {d.fault_id: d for d in reg.all_definitions()} + missing = required - definitions.keys() + assert not missing + assert all(definitions[fid].runbook for fid in required) + assert definitions["PASSF-ACTION-SHALLOW-SUCCESS"].rpn >= 30 + assert definitions["PASSF-FALSE-HEALTH"].rpn >= 30 + class TestFMEARegistry: def test_all_faults_have_fmea_entries(self): @@ -124,6 +147,30 @@ def test_full_report(self): assert "rpn" in entry assert "mitigations" in entry + def test_pass_f_maturity_faults_are_mitigated(self): + from core.resilience.fmea_registry import get_fmea_registry + + fmea = get_fmea_registry() + pass_f = [ + row for row in fmea.full_report() + if row["fault_id"].startswith("PASSF-") + ] + assert len(pass_f) == 10 + assert all(row["mitigated"] for row in pass_f) + assert all(row["runbook"] == "docs/runbooks/pass-f-maturity-risks.md" + for row in pass_f) + assert all(row["mitigation_count"] >= 2 for row in pass_f) + + def test_pass_f_high_risk_faults_surface_in_rpn_report(self): + from core.resilience.fmea_registry import get_fmea_registry + + high_risk_ids = { + row["fault_id"] for row in get_fmea_registry().faults_above_rpn(30) + } + assert "PASSF-ACTION-SHALLOW-SUCCESS" in high_risk_ids + assert "PASSF-FALSE-HEALTH" in high_risk_ids + assert "PASSF-SEMANTIC-REVIEW-GAP" in high_risk_ids + # ═══════════════════════════════════════════════════════════════════════ # Phase 2: Triple Modular Redundancy From 2413c0709e550e76fe49496ef0d6c177fe8da26c Mon Sep 17 00:00:00 2001 From: Zenflow Date: Thu, 9 Jul 2026 18:18:11 -0700 Subject: [PATCH 3/5] Persist action expectation verdict receipts --- core/capability_engine.py | 99 +++++++++++++++++++ docs/AURA_EXECUTION_TRACKER.md | 59 +++++++++++ ...st_capability_engine_policy_regressions.py | 72 ++++++++++++++ 3 files changed, 230 insertions(+) diff --git a/core/capability_engine.py b/core/capability_engine.py index 670e4129b..3c770b848 100644 --- a/core/capability_engine.py +++ b/core/capability_engine.py @@ -1,4 +1,5 @@ import asyncio +import hashlib import importlib import inspect import json @@ -4421,8 +4422,106 @@ def _apply_action_expectation_result( payload["ok"] = checked.ok if not checked.ok and checked.failure_reason and not payload.get("error"): payload["error"] = checked.failure_reason + expectation_receipt_id = cls._emit_action_expectation_receipt( + skill_name, + payload, + expectation, + checked, + ) + if expectation_receipt_id: + payload["expectation_receipt_id"] = expectation_receipt_id + payload["verification_evidence"]["expectation_receipt_id"] = expectation_receipt_id + if isinstance(payload.get("expectation_verdict"), dict): + payload["expectation_verdict"]["receipt_id"] = expectation_receipt_id return payload + @staticmethod + def _action_expectation_digest(payload: dict[str, Any]) -> str: + try: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + except (TypeError, ValueError): + encoded = str(payload).encode("utf-8", errors="replace") + return hashlib.sha256(encoded).hexdigest() + + @classmethod + def _emit_action_expectation_receipt( + cls, + skill_name: str, + result: dict[str, Any], + expectation: Any, + checked: Any, + ) -> str | None: + verdict = checked.verification_evidence.get("expectation_verdict", {}) + if not isinstance(verdict, dict) or not verdict: + return None + + try: + from core.runtime.receipts import ToolExecutionReceipt, get_receipt_store + + receipt = ToolExecutionReceipt( + cause=str(getattr(expectation, "objective", "") or skill_name)[:240], + tool=skill_name, + status=str(checked.status.value), + output_digest=cls._action_expectation_digest( + { + "skill": skill_name, + "status": checked.status.value, + "ok": bool(result.get("ok", False)), + "verdict": verdict, + } + ), + verification_evidence={ + "expectation_verdict": verdict, + "original_receipt_id": checked.receipt_id, + "failure_reason": checked.failure_reason, + }, + metadata={ + "source": "capability_engine.action_expectation", + "expectation_objective": str( + getattr(expectation, "objective", "") or skill_name + )[:240], + "expectation_next_step": str(verdict.get("next_step") or "")[:240], + "passed": bool(verdict.get("passed", False)), + }, + ) + emitted = get_receipt_store().emit(receipt) + if not bool(verdict.get("passed", False)): + try: + from core.resilience.fault_taxonomy import get_fault_registry + + missing = list(verdict.get("missing_criteria") or []) + list( + verdict.get("missing_evidence") or [] + ) + get_fault_registry().record_fault( + "PASSF-ACTION-SHALLOW-SUCCESS", + "capability_engine", + details=( + f"{skill_name} expectation downgraded before verified success; " + f"missing={missing[:6]}" + ), + recovered=True, + recovery_time_s=0.0, + ) + except (ImportError, AttributeError, RuntimeError, TypeError, ValueError) as fault_exc: + _record_capability_degradation( + fault_exc, + action="returned expectation-downgraded result after fault occurrence recording failed", + severity="warning", + ) + return emitted.receipt_id + except (ImportError, OSError, RuntimeError, TypeError, ValueError) as exc: + _record_capability_degradation( + exc, + action="returned expectation verdict after durable receipt emit failed", + severity="warning", + ) + return None + @staticmethod def _outer_retry_disabled( skill_name: str, diff --git a/docs/AURA_EXECUTION_TRACKER.md b/docs/AURA_EXECUTION_TRACKER.md index cc2e2ba73..6efc72029 100644 --- a/docs/AURA_EXECUTION_TRACKER.md +++ b/docs/AURA_EXECUTION_TRACKER.md @@ -8735,3 +8735,62 @@ Remaining work after this checkpoint: - Start the declarative runtime control plane and resource admission layer so high-risk `PASSF-RESOURCE-SPAWN-LOOP` becomes preventable, not just visible. - Keep semantic review coverage moving through the closeout ledger. + +## Checkpoint 2026-07-09-11: Expectation Verdict Durable Receipts + +Scope: + +- Extended `CapabilityEngine._apply_action_expectation_result(...)` so every + action expectation verdict can emit a durable `ToolExecutionReceipt`. +- The returned skill payload now includes `expectation_receipt_id` when receipt + emission succeeds, and the same ID is copied into + `verification_evidence.expectation_receipt_id`. +- Expectation receipts record: + - the skill/tool name + - the final downgraded status + - a stable digest of the verdict payload + - the full `expectation_verdict` + - the original receipt ID when the skill supplied one + - the repair hint / next step +- Failed expectation verdicts now also record a recovered + `PASSF-ACTION-SHALLOW-SUCCESS` fault occurrence. This makes shallow-success + catches visible in the fault taxonomy without turning a successfully + downgraded result into an unrecovered critical fault. +- Added an isolated regression using a `tmp_path` receipt store and stubbed + fault registry to prove the durable receipt and fault occurrence are emitted + without touching live receipt state. + +Verification: + +- `python -m pytest tests/test_capability_engine_policy_regressions.py::test_expectation_downgrade_emits_durable_receipt_and_fault tests/test_capability_engine_policy_regressions.py::test_execute_with_retry_downgrades_shallow_action_expectation tests/test_capability_engine_policy_regressions.py::test_execute_with_retry_marks_missing_expectation_evidence_unverified tests/test_action_depth_honesty.py -q` + -> `20 passed`. +- `python -m pytest tests/test_capability_engine_policy_regressions.py -q` + -> `25 passed`. +- `python -m pytest tests/test_action_depth_honesty.py tests/test_reliability_hardening.py -q` + -> `114 passed`. +- `python -m py_compile core/capability_engine.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `python -m ruff check --select F,E9 core/capability_engine.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `make production-gate` + -> passed; `/tmp/aura_production_readiness.json` has `passed=true` and + `37` checks. +- `make enterprise-gate` + -> passed; `/tmp/aura_enterprise_gate.json` has `counts={}` and + `high_or_critical_count=0`. + +Current closeout estimate: + +- Action-depth enforcement is now durable. A shallow action no longer only + returns a downgraded payload; it leaves a receipt and a fault-taxonomy + occurrence that can be audited and learned from. +- This closes the first half of durable expectation feedback. Remaining work is + to make future planners consult these receipts/fault counts proactively when + choosing action depth. + +Remaining work after this checkpoint: + +- Expand automatic expectation generation beyond desktop tasks. +- Build the declarative runtime control plane and resource admission layer. +- Add planner-side use of recent expectation receipts so repeated shallow + failures change future action planning before execution. diff --git a/tests/test_capability_engine_policy_regressions.py b/tests/test_capability_engine_policy_regressions.py index 40c36f315..cae5e827d 100644 --- a/tests/test_capability_engine_policy_regressions.py +++ b/tests/test_capability_engine_policy_regressions.py @@ -393,6 +393,78 @@ async def safe_execute(self, params, context): ] +@pytest.mark.asyncio +async def test_expectation_downgrade_emits_durable_receipt_and_fault(monkeypatch, tmp_path): + from core.runtime.receipts import get_receipt_store, reset_receipt_store + + reset_receipt_store() + store = get_receipt_store(tmp_path / "receipts") + fault_records = [] + + class FaultRegistryStub: + def record_fault(self, fault_id, subsystem, **kwargs): + fault_records.append((fault_id, subsystem, kwargs)) + + monkeypatch.setattr( + "core.resilience.fault_taxonomy.get_fault_registry", + lambda: FaultRegistryStub(), + ) + + engine = _engine_with_skill("file.write") + engine.max_retries = 1 + + class FileWriteSkill: + async def safe_execute(self, params, context): + return { + "ok": True, + "status": "completed", + "criteria_results": {"file written": True}, + } + + try: + result = await engine._execute_with_retry( + FileWriteSkill(), + "file.write", + {"path": "workspace-note.txt"}, + { + "action_expectation": { + "objective": "write and verify a file", + "acceptance_criteria": ["file written"], + "required_evidence": ["sha256"], + "repair_hint": "hash_file_before_reporting_done", + } + }, + ) + + assert result["ok"] is False + assert result["status"] == "success_unverified" + assert result["expectation_receipt_id"] + assert ( + result["verification_evidence"]["expectation_receipt_id"] + == result["expectation_receipt_id"] + ) + + receipts = store.query_by_kind("tool_execution") + assert len(receipts) == 1 + receipt = receipts[0] + assert receipt.receipt_id == result["expectation_receipt_id"] + assert receipt.tool == "file.write" + assert receipt.status == "success_unverified" + assert receipt.metadata["source"] == "capability_engine.action_expectation" + assert receipt.verification_evidence["expectation_verdict"]["missing_evidence"] == [ + "sha256" + ] + + assert fault_records + fault_id, subsystem, kwargs = fault_records[0] + assert fault_id == "PASSF-ACTION-SHALLOW-SUCCESS" + assert subsystem == "capability_engine" + assert kwargs["recovered"] is True + assert kwargs["recovery_time_s"] == 0.0 + finally: + reset_receipt_store() + + def test_auto_refactor_scan_is_read_only_not_privileged_mutation(): engine = _engine_with_skill("auto_refactor") meta = engine.skills["auto_refactor"] From 90aea3c924c445c3a77020f11b5b9324b230acdc Mon Sep 17 00:00:00 2001 From: Zenflow Date: Thu, 9 Jul 2026 20:05:36 -0700 Subject: [PATCH 4/5] Verify mutating file operation effects --- core/capability_engine.py | 50 ++++++++ core/skills/file_operation.py | 107 +++++++++++++++++- docs/AURA_EXECUTION_TRACKER.md | 66 +++++++++++ tests/test_action_depth_honesty.py | 24 ++++ ...st_capability_engine_policy_regressions.py | 69 +++++++++++ 5 files changed, 310 insertions(+), 6 deletions(-) diff --git a/core/capability_engine.py b/core/capability_engine.py index 3c770b848..f18f0ab2b 100644 --- a/core/capability_engine.py +++ b/core/capability_engine.py @@ -4355,6 +4355,14 @@ def _action_expectation_for( evidence = cls._str_list(source.get("required_evidence") or source.get("evidence_required")) visible_effect = source.get("user_visible_effect") or source.get("visible_effect") if not criteria and not evidence and not visible_effect: + default_expectation = cls._default_action_expectation_for( + skill_name, + params, + context, + ActionExpectation, + ) + if default_expectation is not None: + return default_expectation return None return ActionExpectation( @@ -4374,6 +4382,48 @@ def _action_expectation_for( allow_partial=cls._bool_value(source.get("allow_partial"), default=True), ) + @classmethod + def _default_action_expectation_for( + cls, + skill_name: str, + params: dict[str, Any], + context: dict[str, Any], + expectation_cls: Any, + ) -> Any | None: + if cls._bool_value((context or {}).get("disable_auto_action_expectation"), default=False): + return None + if cls._bool_value((params or {}).get("disable_auto_action_expectation"), default=False): + return None + + normalized_skill = str(skill_name or "").strip().lower() + if normalized_skill != "file_operation": + return None + + action = str((params or {}).get("action") or "").strip().lower() + path = str((params or {}).get("path") or "").strip() + destination = str((params or {}).get("destination") or "").strip() + file_expectations = { + "write": ("file written", ["path", "sha256", "effect_verified"]), + "append": ("file appended", ["path", "sha256", "effect_verified"]), + "patch": ("file patched", ["path", "sha256", "effect_verified"]), + "delete": ("path deleted", ["path", "effect_verified"]), + "move": ("path moved", ["path", "destination", "sha256", "effect_verified"]), + "copy": ("path copied", ["path", "destination", "sha256", "effect_verified"]), + } + if action not in file_expectations: + return None + + criterion, evidence = file_expectations[action] + target = f"{path} -> {destination}" if destination else path + return expectation_cls( + objective=f"{action} file_operation effect for {target or 'requested path'}", + acceptance_criteria=[criterion], + required_evidence=evidence, + user_visible_effect=f"filesystem {action} is observable and verified", + repair_hint=f"verify_file_operation_{action}_effect", + allow_partial=False, + ) + @classmethod def _apply_action_expectation_result( cls, diff --git a/core/skills/file_operation.py b/core/skills/file_operation.py index 908bdd4c3..84a6bf2a1 100644 --- a/core/skills/file_operation.py +++ b/core/skills/file_operation.py @@ -2,6 +2,7 @@ from core.runtime.action_executor import ActionExecutor from core.governance.will import ActionDomain import contextlib +import hashlib import logging import os import tempfile @@ -55,6 +56,41 @@ def _safe_resolve(self, path: str) -> str: raise PermissionError(f"Access denied: path '{path}' resolves outside workspace") return full + @staticmethod + def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + @staticmethod + def _file_sha256(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + @classmethod + async def _file_effect(cls, full_path: str, *, expected_sha256: str = "") -> dict[str, Any]: + import asyncio + + exists = await asyncio.to_thread(os.path.exists, full_path) + is_file = await asyncio.to_thread(os.path.isfile, full_path) if exists else False + evidence: dict[str, Any] = { + "exists": exists, + "is_file": is_file, + "effect_verified": exists, + } + if is_file: + sha256 = await asyncio.to_thread(cls._file_sha256, full_path) + size = await asyncio.to_thread(os.path.getsize, full_path) + evidence.update({"sha256": sha256, "bytes": size}) + if expected_sha256: + evidence["expected_sha256"] = expected_sha256 + evidence["effect_verified"] = sha256 == expected_sha256 + elif expected_sha256: + evidence["expected_sha256"] = expected_sha256 + evidence["effect_verified"] = False + return evidence + def match(self, goal: Dict[str, Any]) -> bool: obj = goal.get("objective", "").lower() return "file" in obj or "read" in obj or "write" in obj or "save" in obj or "log" in obj @@ -138,7 +174,15 @@ def _read(): ) if not result.get("ok"): return {"ok": False, "error": result.get("error", "write failed"), "path": path} - return {"ok": True, "summary": f"Wrote {len(content)} bytes to {path}", "path": path} + expected_sha256 = self._sha256_text(content) + effect = await self._file_effect(full_path, expected_sha256=expected_sha256) + return { + "ok": bool(effect.get("effect_verified")), + "summary": f"Wrote {len(content)} bytes to {path}", + "path": path, + "criteria_results": {"file written": bool(effect.get("effect_verified"))}, + **effect, + } elif action == "append": existing = "" @@ -157,7 +201,15 @@ def _read_existing(): ) if not result.get("ok"): return {"ok": False, "error": result.get("error", "append failed"), "path": path} - return {"ok": True, "summary": f"Appended to {path}", "path": path} + expected_sha256 = self._sha256_text(next_text) + effect = await self._file_effect(full_path, expected_sha256=expected_sha256) + return { + "ok": bool(effect.get("effect_verified")), + "summary": f"Appended to {path}", + "path": path, + "criteria_results": {"file appended": bool(effect.get("effect_verified"))}, + **effect, + } elif action == "list": if await asyncio.to_thread(os.path.isdir, full_path): @@ -199,7 +251,16 @@ def _read_existing(): ) if not result.get("ok"): return {"ok": False, "error": result.get("error", "delete failed"), "path": path} - return {"ok": True, "summary": f"Deleted {path}", "path": path} + exists_after = await asyncio.to_thread(os.path.exists, full_path) + effect_verified = not exists_after + return { + "ok": effect_verified, + "summary": f"Deleted {path}", + "path": path, + "exists": exists_after, + "effect_verified": effect_verified, + "criteria_results": {"path deleted": effect_verified}, + } return {"ok": False, "error": "File not found", "path": path} elif action == "move": @@ -219,7 +280,19 @@ def _read_existing(): ) if not result.get("ok"): return {"ok": False, "error": result.get("error", "move failed"), "path": path} - return {"ok": True, "summary": f"Moved {path} to {dest_path}", "path": path, "destination": dest_path} + source_exists = await asyncio.to_thread(os.path.exists, full_path) + effect = await self._file_effect(full_dest) + effect_verified = bool(effect.get("effect_verified")) and not source_exists + return { + "ok": effect_verified, + "summary": f"Moved {path} to {dest_path}", + "path": path, + "destination": dest_path, + "source_exists": source_exists, + "criteria_results": {"path moved": effect_verified}, + **effect, + "effect_verified": effect_verified, + } elif action == "copy": dest_path = params.destination @@ -238,7 +311,21 @@ def _read_existing(): ) if not result.get("ok"): return {"ok": False, "error": result.get("error", "copy failed"), "path": path} - return {"ok": True, "summary": f"Copied {path} to {dest_path}", "path": path, "destination": dest_path} + source_effect = await self._file_effect(full_path) + dest_effect = await self._file_effect(full_dest) + effect_verified = bool(dest_effect.get("effect_verified")) + if source_effect.get("sha256") and dest_effect.get("sha256"): + effect_verified = source_effect["sha256"] == dest_effect["sha256"] + return { + "ok": effect_verified, + "summary": f"Copied {path} to {dest_path}", + "path": path, + "destination": dest_path, + "source_sha256": source_effect.get("sha256", ""), + "criteria_results": {"path copied": effect_verified}, + **dest_effect, + "effect_verified": effect_verified, + } elif action == "patch": start_line = params.start_line @@ -297,7 +384,15 @@ def _patch(): ) if not result.get("ok"): return {"ok": False, "error": result.get("error", "patch write failed"), "path": path} - return {"ok": True, "summary": f"Patched {path}: Replaced lines {start_line}-{end_line}", "path": path} + expected_sha256 = self._sha256_text(new_content) + effect = await self._file_effect(full_path, expected_sha256=expected_sha256) + return { + "ok": bool(effect.get("effect_verified")), + "summary": f"Patched {path}: Replaced lines {start_line}-{end_line}", + "path": path, + "criteria_results": {"file patched": bool(effect.get("effect_verified"))}, + **effect, + } except ValueError as ve: return {"ok": False, "error": str(ve), "path": path} diff --git a/docs/AURA_EXECUTION_TRACKER.md b/docs/AURA_EXECUTION_TRACKER.md index 6efc72029..8a3814de1 100644 --- a/docs/AURA_EXECUTION_TRACKER.md +++ b/docs/AURA_EXECUTION_TRACKER.md @@ -8794,3 +8794,69 @@ Remaining work after this checkpoint: - Build the declarative runtime control plane and resource admission layer. - Add planner-side use of recent expectation receipts so repeated shallow failures change future action planning before execution. + +## Checkpoint 2026-07-09-12: Verified File Operation Expectations + +Scope: + +- Deepened `file_operation` mutating actions so they verify the filesystem + effect after the governed operation, not just report a summary string. +- `write`, `append`, and `patch` now return: + - `effect_verified` + - `sha256` + - `expected_sha256` + - byte size + - `criteria_results` +- `delete`, `move`, and `copy` now verify post-action state: + - delete confirms the target is gone + - move confirms destination exists and source is gone + - copy compares source/destination hashes when both are files +- Added central automatic `ActionExpectation` generation for mutating + `file_operation` calls in `CapabilityEngine`. A caller no longer has to + remember to pass an explicit expectation for write/append/patch/delete/move + /copy to avoid shallow success. +- Automatic expectations are intentionally not applied to read/list/exists and + can be disabled with `disable_auto_action_expectation` in params/context for + narrow compatibility paths. +- Added regressions proving: + - real `file_operation.write` returns hash-backed effect evidence + - a shallow fake `file_operation.write` result is downgraded automatically + - read-only file actions are not forced through mutation expectations + +Verification: + +- `python -m pytest tests/test_action_depth_honesty.py::test_file_operation_write_returns_effect_evidence tests/test_capability_engine_policy_regressions.py::test_auto_file_operation_expectation_rejects_shallow_mutation tests/test_capability_engine_policy_regressions.py::test_auto_file_operation_expectation_ignores_read_only_actions -q` + -> `3 passed`. +- `python -m pytest tests/test_action_depth_honesty.py tests/test_capability_engine_policy_regressions.py -q` + -> `45 passed`. +- `python -m pytest tests/test_live_runtime_surface_regressions.py::test_file_operation_write_creates_nested_live_runtime_directory -q` + -> `1 passed`. +- `python -m pytest tests/test_skill_surface_contracts.py tests/test_action_depth_honesty.py tests/test_capability_engine_policy_regressions.py -q` + -> `140 passed`. +- `python -m py_compile core/skills/file_operation.py core/capability_engine.py tests/test_action_depth_honesty.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `python -m ruff check --select F,E9 core/skills/file_operation.py core/capability_engine.py tests/test_action_depth_honesty.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `git diff --check` + -> passed. +- `make production-gate` + -> passed; `/tmp/aura_production_readiness.json` has `passed=true` and + `37` checks. +- `make enterprise-gate` + -> passed; `/tmp/aura_enterprise_gate.json` has `counts={}` and + `high_or_critical_count=0`. + +Current closeout estimate: + +- File operations are no longer "action fired" shallow. The core mutating file + skill now proves the post-action filesystem state and the central capability + execution lane auto-enforces that proof for mutating file actions. +- This is the first automatic expectation expansion beyond desktop tasks. Web, + memory, live skill API, and autonomous overt action defaults remain open. + +Remaining work after this checkpoint: + +- Add automatic defaults for memory writes and web research actions where + result shapes are stable enough to avoid noisy false downgrades. +- Add planner-side use of recent expectation receipts. +- Start the runtime control plane/resource admission workstream. diff --git a/tests/test_action_depth_honesty.py b/tests/test_action_depth_honesty.py index 03e77a3de..bd515f92d 100644 --- a/tests/test_action_depth_honesty.py +++ b/tests/test_action_depth_honesty.py @@ -343,6 +343,30 @@ async def scenario(): asyncio.get_event_loop_policy().new_event_loop().run_until_complete(scenario()) +def test_file_operation_write_returns_effect_evidence(tmp_path): + import hashlib + + from core.skills.file_operation import FileOperationSkill + + skill = FileOperationSkill() + skill.root_dir = str(tmp_path.resolve()) + + result = _run( + skill.execute( + {"action": "write", "path": "verified.txt", "content": "real payload"}, + context={"origin": "unit_test"}, + ) + ) + + expected_sha256 = hashlib.sha256(b"real payload").hexdigest() + assert result["ok"] is True + assert result["effect_verified"] is True + assert result["sha256"] == expected_sha256 + assert result["expected_sha256"] == expected_sha256 + assert result["criteria_results"]["file written"] is True + assert (tmp_path / "verified.txt").read_text(encoding="utf-8") == "real payload" + + def test_action_executor_file_ops(tmp_path): from core.runtime.action_executor import ActionExecutor diff --git a/tests/test_capability_engine_policy_regressions.py b/tests/test_capability_engine_policy_regressions.py index cae5e827d..ee75c198b 100644 --- a/tests/test_capability_engine_policy_regressions.py +++ b/tests/test_capability_engine_policy_regressions.py @@ -465,6 +465,75 @@ async def safe_execute(self, params, context): reset_receipt_store() +@pytest.mark.asyncio +async def test_auto_file_operation_expectation_rejects_shallow_mutation(monkeypatch, tmp_path): + from core.runtime.receipts import get_receipt_store, reset_receipt_store + + reset_receipt_store() + get_receipt_store(tmp_path / "receipts") + fault_records = [] + + class FaultRegistryStub: + def record_fault(self, fault_id, subsystem, **kwargs): + fault_records.append((fault_id, subsystem, kwargs)) + + monkeypatch.setattr( + "core.resilience.fault_taxonomy.get_fault_registry", + lambda: FaultRegistryStub(), + ) + + engine = _engine_with_skill("file_operation") + + class ShallowFileSkill: + async def safe_execute(self, params, context): + return {"ok": True, "status": "completed", "path": params["path"]} + + try: + result = await engine._execute_with_retry( + ShallowFileSkill(), + "file_operation", + {"action": "write", "path": "shallow.txt", "content": "x"}, + {"origin": "user"}, + ) + + assert result["ok"] is False + assert result["status"] == "failed_recoverable" + assert result["expectation_verdict"]["missing_criteria"] == [ + "file written", + "user-visible effect: filesystem write is observable and verified", + ] + assert result["expectation_verdict"]["missing_evidence"] == [ + "sha256", + "effect_verified", + ] + assert result["expectation_receipt_id"] + assert any( + fault_id == "PASSF-ACTION-SHALLOW-SUCCESS" + for fault_id, _subsystem, _kwargs in fault_records + ) + finally: + reset_receipt_store() + + +@pytest.mark.asyncio +async def test_auto_file_operation_expectation_ignores_read_only_actions(): + engine = _engine_with_skill("file_operation") + + class ReadFileSkill: + async def safe_execute(self, params, context): + return {"ok": True, "content": "hello", "path": params["path"]} + + result = await engine._execute_with_retry( + ReadFileSkill(), + "file_operation", + {"action": "read", "path": "note.txt"}, + {"origin": "user"}, + ) + + assert result["ok"] is True + assert "expectation_verdict" not in result + + def test_auto_refactor_scan_is_read_only_not_privileged_mutation(): engine = _engine_with_skill("auto_refactor") meta = engine.skills["auto_refactor"] From c934d0dce3a3b3dc4a088e35daf4271a5abde399 Mon Sep 17 00:00:00 2001 From: Zenflow Date: Thu, 9 Jul 2026 20:09:54 -0700 Subject: [PATCH 5/5] Verify core memory mutation effects --- core/capability_engine.py | 19 +++++ core/skills/memory_ops.py | 82 +++++++++++++++++-- docs/AURA_EXECUTION_TRACKER.md | 59 +++++++++++++ ...st_capability_engine_policy_regressions.py | 51 ++++++++++++ tests/test_memory_facade_runtime.py | 5 ++ 5 files changed, 209 insertions(+), 7 deletions(-) diff --git a/core/capability_engine.py b/core/capability_engine.py index f18f0ab2b..956edc843 100644 --- a/core/capability_engine.py +++ b/core/capability_engine.py @@ -4396,6 +4396,25 @@ def _default_action_expectation_for( return None normalized_skill = str(skill_name or "").strip().lower() + if normalized_skill == "memory_ops": + action = str((params or {}).get("action") or "").strip().lower() + memory_expectations = { + "core_append": ("core memory appended", "append"), + "core_replace": ("core memory replaced", "replace"), + } + if action not in memory_expectations: + return None + criterion, verb = memory_expectations[action] + block = str((params or {}).get("block") or "user").strip() + return expectation_cls( + objective=f"{verb} core memory block {block or 'user'}", + acceptance_criteria=[criterion], + required_evidence=["block", "sha256", "effect_verified"], + user_visible_effect=f"core memory {verb} is persisted and verified", + repair_hint=f"verify_memory_ops_{action}_effect", + allow_partial=False, + ) + if normalized_skill != "file_operation": return None diff --git a/core/skills/memory_ops.py b/core/skills/memory_ops.py index 3558abfcb..515a78e07 100644 --- a/core/skills/memory_ops.py +++ b/core/skills/memory_ops.py @@ -2,11 +2,10 @@ from core.runtime.atomic_writer import atomic_write_text from core.runtime.action_executor import ActionExecutor from core.governance.will import ActionDomain +import hashlib import logging -import os -import json from pathlib import Path -from typing import Any, Dict, Optional, List +from typing import Any, Dict, Optional from pydantic import BaseModel, Field from core.config import config @@ -94,6 +93,43 @@ def _normalize_action(cls, action: Any) -> str: def _resolve_memory_facade(context: Dict[str, Any]) -> Any: return context.get("memory_facade") or ServiceContainer.get("memory_facade", default=None) + @staticmethod + def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + @staticmethod + def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + @classmethod + def _core_memory_effect( + cls, + block_path: Path, + *, + expected_sha256: str, + ) -> dict[str, Any]: + exists = block_path.exists() + effect: dict[str, Any] = { + "path": str(block_path), + "exists": exists, + "effect_verified": False, + } + if exists: + sha256 = cls._file_sha256(block_path) + effect.update({ + "sha256": sha256, + "bytes": block_path.stat().st_size, + "expected_sha256": expected_sha256, + "effect_verified": sha256 == expected_sha256, + }) + else: + effect["expected_sha256"] = expected_sha256 + return effect + async def _execute_core_memory(self, params: MemoryOpsInput, context: Dict[str, Any], action: str) -> Dict[str, Any]: """RAM: Immediate context window blocks.""" block = params.block or "user" @@ -110,13 +146,29 @@ async def _execute_core_memory(self, params: MemoryOpsInput, context: Dict[str, with open(block_path, "r", encoding="utf-8") as f: current_content = f.read() new_content = current_content + params.content + "\n" - await ActionExecutor.execute( + result = await ActionExecutor.execute( domain=ActionDomain.FILE_WRITE, action_name="core_append", params={"path": str(block_path), "text": new_content}, source="memory_ops", ) - return {"ok": True, "summary": f"Appended to core memory block '{block}'."} + if not result.get("ok"): + return { + "ok": False, + "error": result.get("error", "core memory append failed"), + "block": block, + } + effect = self._core_memory_effect( + block_path, + expected_sha256=self._sha256_text(new_content), + ) + return { + "ok": bool(effect.get("effect_verified")), + "summary": f"Appended to core memory block '{block}'.", + "block": block, + "criteria_results": {"core memory appended": bool(effect.get("effect_verified"))}, + **effect, + } elif action == "core_replace": if not params.content or not params.old_content: @@ -129,13 +181,29 @@ async def _execute_core_memory(self, params: MemoryOpsInput, context: Dict[str, return {"ok": False, "error": f"Text to replace not found in block '{block}'."} new_data = data.replace(params.old_content, params.content) - await ActionExecutor.execute( + result = await ActionExecutor.execute( domain=ActionDomain.FILE_WRITE, action_name="core_replace", params={"path": str(block_path), "text": new_data}, source="memory_ops", ) - return {"ok": True, "summary": f"Replaced content in core memory block '{block}'."} + if not result.get("ok"): + return { + "ok": False, + "error": result.get("error", "core memory replace failed"), + "block": block, + } + effect = self._core_memory_effect( + block_path, + expected_sha256=self._sha256_text(new_data), + ) + return { + "ok": bool(effect.get("effect_verified")), + "summary": f"Replaced content in core memory block '{block}'.", + "block": block, + "criteria_results": {"core memory replaced": bool(effect.get("effect_verified"))}, + **effect, + } return {"ok": False, "error": f"Unknown core action: {action}"} diff --git a/docs/AURA_EXECUTION_TRACKER.md b/docs/AURA_EXECUTION_TRACKER.md index 8a3814de1..3c261e406 100644 --- a/docs/AURA_EXECUTION_TRACKER.md +++ b/docs/AURA_EXECUTION_TRACKER.md @@ -8860,3 +8860,62 @@ Remaining work after this checkpoint: result shapes are stable enough to avoid noisy false downgrades. - Add planner-side use of recent expectation receipts. - Start the runtime control plane/resource admission workstream. + +## Checkpoint 2026-07-09-13: Verified Core Memory Expectations + +Scope: + +- Deepened `memory_ops` core-memory mutations so `core_append` and + `core_replace` verify the persisted MemFS block after the governed write. +- `memory_ops` now checks the `ActionExecutor` result instead of assuming the + write succeeded. +- Successful core-memory mutations now return: + - `block` + - `path` + - `effect_verified` + - `sha256` + - `expected_sha256` + - byte size + - `criteria_results` +- Added central automatic `ActionExpectation` generation for `memory_ops` + `core_append` and `core_replace`. Shallow memory writes can no longer claim + success without block identity, hash evidence, and effect verification. +- Left archival memory insert/search out of automatic defaults for now because + those result shapes depend on the live memory backend/gateway and need a + separate stable receipt contract before enforcement. + +Verification: + +- `python -m pytest tests/test_memory_facade_runtime.py::test_memory_ops_core_append_writes_to_block tests/test_capability_engine_policy_regressions.py::test_auto_memory_ops_expectation_rejects_shallow_core_append -q` + -> `2 passed`. +- `python -m pytest tests/test_memory_facade_runtime.py tests/test_capability_engine_policy_regressions.py -q` + -> `56 passed`. +- `python -m pytest tests/test_memory_facade_runtime.py tests/test_capability_engine_policy_regressions.py tests/test_skill_surface_contracts.py -q` + -> `151 passed`. +- `python -m py_compile core/skills/memory_ops.py core/capability_engine.py tests/test_memory_facade_runtime.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `python -m ruff check --select F,E9 core/skills/memory_ops.py core/capability_engine.py tests/test_memory_facade_runtime.py tests/test_capability_engine_policy_regressions.py` + -> passed. +- `git diff --check` + -> passed. +- `make production-gate` + -> passed; `/tmp/aura_production_readiness.json` has `passed=true` and + `37` checks. +- `make enterprise-gate` + -> passed; `/tmp/aura_enterprise_gate.json` has `counts={}` and + `high_or_critical_count=0`. + +Current closeout estimate: + +- Automatic expectation enforcement now covers desktop tasks, mutating file + operations, and core-memory block mutations. These are the highest-confidence + stateful local effects because they can be verified deterministically. +- Web research defaults and archival-memory receipt stabilization remain open. + +Remaining work after this checkpoint: + +- Stabilize archival memory insert receipts before adding automatic archival + memory expectations. +- Add automatic web research expectations requiring sources/citations for + research-shaped web tasks. +- Add planner-side use of recent expectation receipts. diff --git a/tests/test_capability_engine_policy_regressions.py b/tests/test_capability_engine_policy_regressions.py index ee75c198b..91e443f8e 100644 --- a/tests/test_capability_engine_policy_regressions.py +++ b/tests/test_capability_engine_policy_regressions.py @@ -534,6 +534,57 @@ async def safe_execute(self, params, context): assert "expectation_verdict" not in result +@pytest.mark.asyncio +async def test_auto_memory_ops_expectation_rejects_shallow_core_append(monkeypatch, tmp_path): + from core.runtime.receipts import get_receipt_store, reset_receipt_store + + reset_receipt_store() + get_receipt_store(tmp_path / "receipts") + fault_records = [] + + class FaultRegistryStub: + def record_fault(self, fault_id, subsystem, **kwargs): + fault_records.append((fault_id, subsystem, kwargs)) + + monkeypatch.setattr( + "core.resilience.fault_taxonomy.get_fault_registry", + lambda: FaultRegistryStub(), + ) + + engine = _engine_with_skill("memory_ops") + + class ShallowMemorySkill: + async def safe_execute(self, params, context): + return {"ok": True, "summary": "Appended."} + + try: + result = await engine._execute_with_retry( + ShallowMemorySkill(), + "memory_ops", + {"action": "core_append", "block": "user", "content": "remember this"}, + {"origin": "user"}, + ) + + assert result["ok"] is False + assert result["status"] == "failed_recoverable" + assert result["expectation_verdict"]["missing_criteria"] == [ + "core memory appended", + "user-visible effect: core memory append is persisted and verified", + ] + assert result["expectation_verdict"]["missing_evidence"] == [ + "block", + "sha256", + "effect_verified", + ] + assert result["expectation_receipt_id"] + assert any( + fault_id == "PASSF-ACTION-SHALLOW-SUCCESS" + for fault_id, _subsystem, _kwargs in fault_records + ) + finally: + reset_receipt_store() + + def test_auto_refactor_scan_is_read_only_not_privileged_mutation(): engine = _engine_with_skill("auto_refactor") meta = engine.skills["auto_refactor"] diff --git a/tests/test_memory_facade_runtime.py b/tests/test_memory_facade_runtime.py index 4c2040928..2ca7214b1 100644 --- a/tests/test_memory_facade_runtime.py +++ b/tests/test_memory_facade_runtime.py @@ -1,5 +1,6 @@ import json import asyncio +import hashlib import time from types import SimpleNamespace @@ -538,6 +539,10 @@ async def test_memory_ops_core_append_writes_to_block(tmp_path, monkeypatch): assert "user" in result["summary"] block_text = (skill.mem_fs_dir / "user.txt").read_text() assert "verification_codename: glass orchard" in block_text + assert result["effect_verified"] is True + assert result["block"] == "user" + assert result["sha256"] == hashlib.sha256(block_text.encode("utf-8")).hexdigest() + assert result["criteria_results"]["core memory appended"] is True @pytest.mark.asyncio